diff --git a/dev-support/findbugs-exclude.xml b/dev-support/findbugs-exclude.xml index 4d17582..8f9bf74 100644 --- a/dev-support/findbugs-exclude.xml +++ b/dev-support/findbugs-exclude.xml @@ -257,7 +257,7 @@ - + diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestFavoredNodes.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestFavoredNodes.java new file mode 100644 index 0000000..63272b5 --- /dev/null +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestFavoredNodes.java @@ -0,0 +1,420 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; +import java.util.Set; + +import com.google.common.net.HostAndPort; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.rsgroup.RSGroupAdmin; +import org.apache.hadoop.hbase.rsgroup.RSGroupAdminEndpoint; +import org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer; +import org.apache.hadoop.hbase.rsgroup.RSGroupInfo; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.master.SnapshotOfRegionAssignmentFromMeta; +import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan; +import org.apache.hadoop.hbase.master.balancer.FavoredStochasticBalancer; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.testclassification.IntegrationTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.LoadTestTool; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.util.ToolRunner; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.base.Joiner; +import com.google.common.collect.Sets; + +@Category(IntegrationTests.class) +public class IntegrationTestFavoredNodes extends IntegrationTestBase { + protected static final Log LOG = LogFactory + .getLog(IntegrationTestFavoredNodes.class); + private Configuration conf; + private static final int SERVER_COUNT = 5; // number of slaves for the smallest cluster + private Random random = new Random(); + private List regionsWithData; + private LoadTestTool loadTool; + private HBaseAdmin admin; + private RSGroupAdmin groupAdmin; + + /** A soft limit on how long we should run */ + private static final String NAMESPACE = "integration_test"; + private static final String GROUP_NAME = "testgroup"; + private static final String TABLE_NAME = NAMESPACE + TableName.NAMESPACE_DELIM + "balance_with_fav_nodes"; + private static final String FAIL_ON_ERROR = "hbase.testfavnodes.failonerror"; + private static final String MAX_ERRORS = "hbase.testfavnodes.max.errors"; + private static final long DEFAULT_RUN_TIME = 5 * 60 * 1000; + + @Override + public void setUpCluster() throws Exception { + conf = HBaseConfiguration.create(); + conf.setClass(RSGroupBasedLoadBalancer.HBASE_GROUP_LOADBALANCER_CLASS, + FavoredStochasticBalancer.class, LoadBalancer.class); + conf.set("hbase.coprocessor.master.classes", RSGroupAdminEndpoint.class.getName()); + regionsWithData = new ArrayList(); + util = getTestingUtil(conf); + LOG.debug("Initializing/checking cluster has " + SERVER_COUNT + " servers"); + util.initializeCluster(SERVER_COUNT); + LOG.debug("Done initializing/checking cluster"); + admin = util.getHBaseAdmin(); + groupAdmin = RSGroupAdmin.newClient(admin.getConnection()); + deleteTableIfNecessary(); + loadTool = new LoadTestTool(); + loadTool.setConf(conf); + initTable(); + } + + @Override + public void cleanUpCluster() throws Exception { + deleteTableIfNecessary(); + admin.deleteNamespace(NAMESPACE); + RSGroupInfo groupInfo = groupAdmin.getRSGroupInfo(GROUP_NAME); + groupAdmin.moveTables(groupInfo.getTables(), RSGroupInfo.DEFAULT_GROUP); + groupAdmin.moveServers(groupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP); + groupAdmin.removeRSGroup(GROUP_NAME); + super.cleanUpCluster(); + } + + protected void initTable() throws IOException, InterruptedException { + addGroup(groupAdmin, GROUP_NAME, 4); + NamespaceDescriptor nspDesc = NamespaceDescriptor.create(NAMESPACE) + .addConfiguration(RSGroupInfo.NAMESPACEDESC_PROP_GROUP, GROUP_NAME).build(); + admin.createNamespace(nspDesc); + int ret = loadTool.run(new String[] { "-tn", TABLE_NAME, "-init_only" }); + Assert.assertEquals("Failed to initialize LoadTestTool", 0, ret); + } + + @Override + public Configuration getConf() { + return conf; + } + + private void deleteTableIfNecessary() throws IOException { + TableName tableName = TableName.valueOf(TABLE_NAME); + if (util.getHBaseAdmin().tableExists(tableName)) { + util.deleteTable(tableName); + } + } + + @Override + public int runTestFromCommandLine() throws Exception { + generateDataAndVerifyFavNodes(DEFAULT_RUN_TIME); + return 1; + } + + @Test + public void testFavNodes() throws Exception { + generateDataAndVerifyFavNodes(DEFAULT_RUN_TIME); + } + + @Override + public TableName getTablename() { + return TableName.valueOf(TABLE_NAME); + } + + @Override + protected Set getColumnFamilies() { + return Sets.newHashSet(Bytes.toString(LoadTestTool.DEFAULT_COLUMN_FAMILY)); + } + + public static void main(String[] args) throws Exception { + Configuration conf = HBaseConfiguration.create(); + IntegrationTestingUtility.setUseDistributedCluster(conf); + int ret = ToolRunner.run(conf, new IntegrationTestFavoredNodes(), args); + System.exit(ret); + } + + private void generateDataAndVerifyFavNodes(long runTime) throws Exception { + LOG.info("Running IntegrationTestFavoredStochasticBalancer"); + generateData(runTime); + Map> regionLocalityMap = FSUtils + .getRegionDegreeLocalityMappingFromFS(getConf(), getTablename().getNameAsString(), 2); + assertLocality100Percent(regionLocalityMap); + buildRegionsWithData(regionLocalityMap.keySet()); + + //Verify META has all the favored nodes information. + SnapshotOfRegionAssignmentFromMeta metaSnapshot = new SnapshotOfRegionAssignmentFromMeta( + admin.getConnection()); + metaSnapshot.initialize(); + Map> favoredNodesInfo = metaSnapshot.getExistingAssignmentPlan() + .getAssignmentMap(); + List regionsOfTable = util.getHBaseAdmin().getTableRegions( + getTablename()); + assertTrue(favoredNodesInfo.size() >= regionsOfTable.size()); + for (HRegionInfo rInfo : regionsOfTable) { + List favNodes = favoredNodesInfo.get(rInfo); + assertNotNull(favNodes); + assertTrue("3 favored nodes not found.", favNodes.size() == 3); + } + + //Test locality with move to favored nodes server. + HRegionInfo regionToBeMoved = getRandomRegion(); + assertNotNull(regionToBeMoved); + ServerName currentRS = util.getHBaseClusterInterface().getServerHoldingRegion( + getTablename(), regionToBeMoved.getRegionName()); + verifyMoveToFavoredNodes(regionToBeMoved, favoredNodesInfo.get(regionToBeMoved), currentRS); + + LOG.info("Killing region server:" + currentRS); + util.getHBaseClusterInterface().killRegionServer(currentRS); + LOG.info("Sleeping for 2 minutes"); + Threads.sleep(120000); + + Map localityMap = new HashMap(); + float localitySum = 0; + for (ServerName sn : util.getHBaseAdmin().getClusterStatus().getServers()) { + float locality = getPercentFilesLocal(sn); + localitySum += locality; + localityMap.put(sn, locality); + } + float avgLocalityBeforeBalance = (float)localitySum/localityMap.size(); + + LOG.info("Balancing cluster"); + util.getHBaseAdmin().balancer(); + LOG.info("Sleeping for 3 minutes"); + Threads.sleep(180000); + localitySum = 0; + Map localityMapAfterBalance = new HashMap(); + for (ServerName sn : util.getHBaseAdmin().getClusterStatus().getServers()) { + float locality = getPercentFilesLocal(sn); + localitySum += locality; + localityMapAfterBalance.put(sn, locality); + } + float avgLocalityAfterBalance = (float) localitySum/localityMapAfterBalance.size(); + + Joiner.MapJoiner mapJoiner = Joiner.on(',').withKeyValueSeparator("="); + LOG.info("Servers before and after balance different: Before " + mapJoiner.join(localityMap) + + " After " + mapJoiner.join(localityMapAfterBalance)); + assertTrue(localityMap.size() == localityMapAfterBalance.size()); + LOG.info("Verifying locality improvement after balance"); + for (ServerName sn : util.getHBaseAdmin().getClusterStatus().getServers()) { + assertTrue("Server not found in before balance locality map " + sn, + localityMap.containsKey(sn)); + assertTrue("Server not found in after balance locality map " + sn, + localityMapAfterBalance.containsKey(sn)); + LOG.info("Server = " + sn + " Before balance locality " + localityMap.get(sn) + + " After balance locality " + localityMapAfterBalance.get(sn)); + } + + assertTrue("Locality before " + avgLocalityBeforeBalance + " after balance locality " + + avgLocalityAfterBalance, avgLocalityAfterBalance >= avgLocalityBeforeBalance); + + } + + private void generateData(long runTime) throws IOException, InterruptedException { + LOG.info("Cluster size:" + util.getHBaseClusterInterface().getClusterStatus().getServersSize()); + long start = System.currentTimeMillis(); + boolean failOnError = util.getConfiguration().getBoolean(FAIL_ON_ERROR, true); + long startKey = 0; + long numKeys = getNumKeys(2500); + while (System.currentTimeMillis() - start < 0.9 * runTime) { + LOG.info("Intended run time: " + (runTime/60000) + " min, left:" + + ((runTime - (System.currentTimeMillis() - start))/60000) + " min"); + int ret = -1; + List args = new ArrayList(); + args.add("-tn"); + args.add(getTablename().getNameAsString()); + args.add("-write"); + args.add(String.format("%d:%d:%d", 10, 1024, 10)); + args.add("-start_key"); + args.add(String.valueOf(startKey)); + args.add("-num_keys"); + args.add(String.valueOf(numKeys)); + args.add("-skip_init"); + args.add("-max_read_errors"); + int maxErrors = Integer.parseInt(this.util.getConfiguration().get(MAX_ERRORS, + Integer.MAX_VALUE + "")); + args.add(String.valueOf(maxErrors)); + ret = loadTool.run(args.toArray(new String[args.size()])); + if (0 != ret) { + String errorMsg = "Load failed with error code " + ret; + LOG.error(errorMsg); + if (failOnError) { + Assert.fail(errorMsg); + } + } + } + util.getHBaseAdmin().flush(getTablename()); + } + + private void moveRegionToDestination(final HRegionInfo regionToBeMoved, final ServerName destination) + throws Exception { + LOG.debug("Region to be moved : " + regionToBeMoved + " to destination " + destination); + util.getHBaseAdmin().move(regionToBeMoved.getEncodedNameAsBytes(), + Bytes.toBytes(destination.getServerName())); + util.waitFor(60000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return (util.getHBaseAdmin().getOnlineRegions(destination).contains(regionToBeMoved)); + } + }); + /* assertTrue("Region not moved to expected server : " + destination, util.getHBaseAdmin() + .getOnlineRegions(destination).contains(regionToBeMoved));*/ + } + + private void assertLocality100Percent(Map> regionLocalityMap) + throws IOException { + LOG.debug("Found " + regionLocalityMap.size() + " entries in regionLocalityMap."); + for (Entry> entry : regionLocalityMap.entrySet()) { + Map localityIndex = entry.getValue(); + assertTrue("Size of region servers holding region not 3 but " + localityIndex.size() + + ". Region encoded name : " + entry.getKey(), localityIndex.size() == 3); + for (String sName : localityIndex.keySet()) { + assertTrue("Locality not 100 % ", localityIndex.get(sName) == 1); + } + } + } + + private ServerName getUnrelatedServer(List servers) throws IOException { + Set onlineServers = new HashSet(); + onlineServers.addAll(util.getHBaseClusterInterface().getClusterStatus().getServers()); + LOG.debug("Online servers are : "); + for (ServerName sn : onlineServers) { + LOG.debug(sn); + } + LOG.debug(" Favored region servers are : "); + for (ServerName sn : servers) { + LOG.debug(sn); + } + ArrayList finalList = new ArrayList(); + finalList.addAll(onlineServers); + for (ServerName server : servers) { + for (ServerName curr : onlineServers) { + if (ServerName.isSameHostnameAndPort(curr, server)) { + finalList.remove(curr); + } + } + } + return finalList.get(random.nextInt(finalList.size())); + } + + private void verifyMoveToFavoredNodes(HRegionInfo region, List favNodes, + ServerName currentRS) throws Exception { + FavoredNodesPlan.Position favoredNodePosition = FavoredNodesPlan.getFavoredServerPosition( + favNodes, currentRS); + assertTrue("Region not on favored node : " + region.getEncodedName(), + favoredNodePosition != null); + // Move to the other remaining positions and verify locality is 100 % + for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) { + ServerName sn = favNodes.get(p.ordinal()); + if ((p.compareTo(favoredNodePosition) != 0) + && (!ServerName.isSameHostnameAndPort(sn, + RSGroupBasedLoadBalancer.BOGUS_SERVER_NAME))) { + ServerName favoredNode = getServerNameWithCorrectStartCode(sn); + assertNotNull("Favored node not among online region servers", favoredNode); + Float originalLocality = getPercentFilesLocal(favoredNode); + moveRegionToDestination(region, favoredNode); + Float locality = getPercentFilesLocal(favoredNode); + assertTrue("Locality on favored node did not improve, ServerName : " + favoredNode + + " Region : " + region, locality >= originalLocality); + } + } + } + + private HRegionInfo getRandomRegion() { + if (regionsWithData.size() > 0) { + return regionsWithData.get(random.nextInt(regionsWithData.size())); + } else { + LOG.warn("No regions with data found in table."); + return null; + } + } + + /** + * Builds the list of regions with storefiles. This is needed + * to check locality after region move. + * + * @param regions List of region encoded names. + * @throws IOException Signals that an I/O exception has occurred. + */ + private void buildRegionsWithData(Set regions) throws IOException { + List regionsOfTable = util.getHBaseAdmin().getTableRegions( + TableName.valueOf(TABLE_NAME)); + for (HRegionInfo info : regionsOfTable) { + if (regions.contains(info.getEncodedName())) { + regionsWithData.add(info); + } + } + } + + private ServerName getServerNameWithCorrectStartCode(ServerName sn) throws IOException { + Collection servers = util.getHBaseAdmin().getClusterStatus().getServers(); + for (ServerName sName : servers) { + if (ServerName.isSameHostnameAndPort(sName, sn)) { + return sName; + } + } + return null; + } + + private float getPercentFilesLocal(ServerName sn) throws IOException { + List regionOfServer = util.getHBaseAdmin().getOnlineRegions(sn); + HDFSBlocksDistribution blockDist = new HDFSBlocksDistribution(); + HTableDescriptor desc = util.getHBaseAdmin().getTableDescriptor(TableName.valueOf(TABLE_NAME)); + for (HRegionInfo regionInfo : regionOfServer) { + blockDist.add(HRegion.computeHDFSBlocksDistribution(getConf(), desc, regionInfo)); + } + return blockDist.getBlockLocalityIndex(sn.getHostname()); + } + + /** Estimates a data size based on the cluster size */ + private long getNumKeys(int keysPerServer) + throws IOException { + int numRegionServers = util.getHBaseAdmin().getClusterStatus().getServersSize(); + return keysPerServer * numRegionServers; + } + + private RSGroupInfo addGroup(RSGroupAdmin gAdmin, String groupName, int serverCount) + throws IOException, InterruptedException { + RSGroupInfo defaultInfo = gAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP); + assertTrue(defaultInfo != null); + assertTrue(defaultInfo.getServers().size() >= serverCount); + gAdmin.addRSGroup(groupName); + + Set set = new HashSet(); + for (HostAndPort server : defaultInfo.getServers()) { + if (set.size() == serverCount) { + break; + } + set.add(server); + } + gAdmin.moveServers(set, groupName); + RSGroupInfo result = gAdmin.getRSGroupInfo(groupName); + assertTrue(result.getServers().size() >= serverCount); + return result; + } + +} diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RSGroupAdminProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RSGroupAdminProtos.java index 3d1f4bd..3edbce9 100644 --- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RSGroupAdminProtos.java +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RSGroupAdminProtos.java @@ -10754,6 +10754,3252 @@ public final class RSGroupAdminProtos { // @@protoc_insertion_point(class_scope:hbase.pb.GetRSGroupInfoOfServerResponse) } + public interface RedistributeGroupRequestOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required string group_name = 1; + /** + * required string group_name = 1; + */ + boolean hasGroupName(); + /** + * required string group_name = 1; + */ + java.lang.String getGroupName(); + /** + * required string group_name = 1; + */ + com.google.protobuf.ByteString + getGroupNameBytes(); + } + /** + * Protobuf type {@code hbase.pb.RedistributeGroupRequest} + */ + public static final class RedistributeGroupRequest extends + com.google.protobuf.GeneratedMessage + implements RedistributeGroupRequestOrBuilder { + // Use RedistributeGroupRequest.newBuilder() to construct. + private RedistributeGroupRequest(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private RedistributeGroupRequest(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final RedistributeGroupRequest defaultInstance; + public static RedistributeGroupRequest getDefaultInstance() { + return defaultInstance; + } + + public RedistributeGroupRequest getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private RedistributeGroupRequest( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + groupName_ = input.readBytes(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_RedistributeGroupRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_RedistributeGroupRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest.class, org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public RedistributeGroupRequest parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new RedistributeGroupRequest(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + private int bitField0_; + // required string group_name = 1; + public static final int GROUP_NAME_FIELD_NUMBER = 1; + private java.lang.Object groupName_; + /** + * required string group_name = 1; + */ + public boolean hasGroupName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * required string group_name = 1; + */ + public java.lang.String getGroupName() { + java.lang.Object ref = groupName_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupName_ = s; + } + return s; + } + } + /** + * required string group_name = 1; + */ + public com.google.protobuf.ByteString + getGroupNameBytes() { + java.lang.Object ref = groupName_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + groupName_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private void initFields() { + groupName_ = ""; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasGroupName()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getGroupNameBytes()); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getGroupNameBytes()); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest)) { + return super.equals(obj); + } + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest other = (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest) obj; + + boolean result = true; + result = result && (hasGroupName() == other.hasGroupName()); + if (hasGroupName()) { + result = result && getGroupName() + .equals(other.getGroupName()); + } + result = result && + getUnknownFields().equals(other.getUnknownFields()); + return result; + } + + private int memoizedHashCode = 0; + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptorForType().hashCode(); + if (hasGroupName()) { + hash = (37 * hash) + GROUP_NAME_FIELD_NUMBER; + hash = (53 * hash) + getGroupName().hashCode(); + } + hash = (29 * hash) + getUnknownFields().hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code hbase.pb.RedistributeGroupRequest} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_RedistributeGroupRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_RedistributeGroupRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest.class, org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest.Builder.class); + } + + // Construct using org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + groupName_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_RedistributeGroupRequest_descriptor; + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest getDefaultInstanceForType() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest build() { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest buildPartial() { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest result = new org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupName_ = groupName_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest) { + return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest other) { + if (other == org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest.getDefaultInstance()) return this; + if (other.hasGroupName()) { + bitField0_ |= 0x00000001; + groupName_ = other.groupName_; + onChanged(); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupName()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // required string group_name = 1; + private java.lang.Object groupName_ = ""; + /** + * required string group_name = 1; + */ + public boolean hasGroupName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * required string group_name = 1; + */ + public java.lang.String getGroupName() { + java.lang.Object ref = groupName_; + if (!(ref instanceof java.lang.String)) { + java.lang.String s = ((com.google.protobuf.ByteString) ref) + .toStringUtf8(); + groupName_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * required string group_name = 1; + */ + public com.google.protobuf.ByteString + getGroupNameBytes() { + java.lang.Object ref = groupName_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + groupName_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * required string group_name = 1; + */ + public Builder setGroupName( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupName_ = value; + onChanged(); + return this; + } + /** + * required string group_name = 1; + */ + public Builder clearGroupName() { + bitField0_ = (bitField0_ & ~0x00000001); + groupName_ = getDefaultInstance().getGroupName(); + onChanged(); + return this; + } + /** + * required string group_name = 1; + */ + public Builder setGroupNameBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupName_ = value; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:hbase.pb.RedistributeGroupRequest) + } + + static { + defaultInstance = new RedistributeGroupRequest(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:hbase.pb.RedistributeGroupRequest) + } + + public interface RedistributeGroupResponseOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required bool result = 1; + /** + * required bool result = 1; + */ + boolean hasResult(); + /** + * required bool result = 1; + */ + boolean getResult(); + } + /** + * Protobuf type {@code hbase.pb.RedistributeGroupResponse} + */ + public static final class RedistributeGroupResponse extends + com.google.protobuf.GeneratedMessage + implements RedistributeGroupResponseOrBuilder { + // Use RedistributeGroupResponse.newBuilder() to construct. + private RedistributeGroupResponse(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private RedistributeGroupResponse(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final RedistributeGroupResponse defaultInstance; + public static RedistributeGroupResponse getDefaultInstance() { + return defaultInstance; + } + + public RedistributeGroupResponse getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private RedistributeGroupResponse( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + result_ = input.readBool(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_RedistributeGroupResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_RedistributeGroupResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse.class, org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public RedistributeGroupResponse parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new RedistributeGroupResponse(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + private int bitField0_; + // required bool result = 1; + public static final int RESULT_FIELD_NUMBER = 1; + private boolean result_; + /** + * required bool result = 1; + */ + public boolean hasResult() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * required bool result = 1; + */ + public boolean getResult() { + return result_; + } + + private void initFields() { + result_ = false; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasResult()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBool(1, result_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(1, result_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse)) { + return super.equals(obj); + } + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse other = (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse) obj; + + boolean result = true; + result = result && (hasResult() == other.hasResult()); + if (hasResult()) { + result = result && (getResult() + == other.getResult()); + } + result = result && + getUnknownFields().equals(other.getUnknownFields()); + return result; + } + + private int memoizedHashCode = 0; + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptorForType().hashCode(); + if (hasResult()) { + hash = (37 * hash) + RESULT_FIELD_NUMBER; + hash = (53 * hash) + hashBoolean(getResult()); + } + hash = (29 * hash) + getUnknownFields().hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code hbase.pb.RedistributeGroupResponse} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponseOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_RedistributeGroupResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_RedistributeGroupResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse.class, org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse.Builder.class); + } + + // Construct using org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + result_ = false; + bitField0_ = (bitField0_ & ~0x00000001); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_RedistributeGroupResponse_descriptor; + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse getDefaultInstanceForType() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse build() { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse buildPartial() { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse result = new org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.result_ = result_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse) { + return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse other) { + if (other == org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse.getDefaultInstance()) return this; + if (other.hasResult()) { + setResult(other.getResult()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasResult()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // required bool result = 1; + private boolean result_ ; + /** + * required bool result = 1; + */ + public boolean hasResult() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * required bool result = 1; + */ + public boolean getResult() { + return result_; + } + /** + * required bool result = 1; + */ + public Builder setResult(boolean value) { + bitField0_ |= 0x00000001; + result_ = value; + onChanged(); + return this; + } + /** + * required bool result = 1; + */ + public Builder clearResult() { + bitField0_ = (bitField0_ & ~0x00000001); + result_ = false; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:hbase.pb.RedistributeGroupResponse) + } + + static { + defaultInstance = new RedistributeGroupResponse(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:hbase.pb.RedistributeGroupResponse) + } + + public interface CompleteRedistributeGroupRequestOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required string group_name = 1; + /** + * required string group_name = 1; + */ + boolean hasGroupName(); + /** + * required string group_name = 1; + */ + java.lang.String getGroupName(); + /** + * required string group_name = 1; + */ + com.google.protobuf.ByteString + getGroupNameBytes(); + } + /** + * Protobuf type {@code hbase.pb.CompleteRedistributeGroupRequest} + */ + public static final class CompleteRedistributeGroupRequest extends + com.google.protobuf.GeneratedMessage + implements CompleteRedistributeGroupRequestOrBuilder { + // Use CompleteRedistributeGroupRequest.newBuilder() to construct. + private CompleteRedistributeGroupRequest(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private CompleteRedistributeGroupRequest(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final CompleteRedistributeGroupRequest defaultInstance; + public static CompleteRedistributeGroupRequest getDefaultInstance() { + return defaultInstance; + } + + public CompleteRedistributeGroupRequest getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private CompleteRedistributeGroupRequest( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + groupName_ = input.readBytes(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CompleteRedistributeGroupRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CompleteRedistributeGroupRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest.class, org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public CompleteRedistributeGroupRequest parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new CompleteRedistributeGroupRequest(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + private int bitField0_; + // required string group_name = 1; + public static final int GROUP_NAME_FIELD_NUMBER = 1; + private java.lang.Object groupName_; + /** + * required string group_name = 1; + */ + public boolean hasGroupName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * required string group_name = 1; + */ + public java.lang.String getGroupName() { + java.lang.Object ref = groupName_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupName_ = s; + } + return s; + } + } + /** + * required string group_name = 1; + */ + public com.google.protobuf.ByteString + getGroupNameBytes() { + java.lang.Object ref = groupName_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + groupName_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private void initFields() { + groupName_ = ""; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasGroupName()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getGroupNameBytes()); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getGroupNameBytes()); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest)) { + return super.equals(obj); + } + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest other = (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest) obj; + + boolean result = true; + result = result && (hasGroupName() == other.hasGroupName()); + if (hasGroupName()) { + result = result && getGroupName() + .equals(other.getGroupName()); + } + result = result && + getUnknownFields().equals(other.getUnknownFields()); + return result; + } + + private int memoizedHashCode = 0; + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptorForType().hashCode(); + if (hasGroupName()) { + hash = (37 * hash) + GROUP_NAME_FIELD_NUMBER; + hash = (53 * hash) + getGroupName().hashCode(); + } + hash = (29 * hash) + getUnknownFields().hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code hbase.pb.CompleteRedistributeGroupRequest} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CompleteRedistributeGroupRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CompleteRedistributeGroupRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest.class, org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest.Builder.class); + } + + // Construct using org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + groupName_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CompleteRedistributeGroupRequest_descriptor; + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest getDefaultInstanceForType() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest build() { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest buildPartial() { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest result = new org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupName_ = groupName_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest) { + return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest other) { + if (other == org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest.getDefaultInstance()) return this; + if (other.hasGroupName()) { + bitField0_ |= 0x00000001; + groupName_ = other.groupName_; + onChanged(); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupName()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // required string group_name = 1; + private java.lang.Object groupName_ = ""; + /** + * required string group_name = 1; + */ + public boolean hasGroupName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * required string group_name = 1; + */ + public java.lang.String getGroupName() { + java.lang.Object ref = groupName_; + if (!(ref instanceof java.lang.String)) { + java.lang.String s = ((com.google.protobuf.ByteString) ref) + .toStringUtf8(); + groupName_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * required string group_name = 1; + */ + public com.google.protobuf.ByteString + getGroupNameBytes() { + java.lang.Object ref = groupName_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + groupName_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * required string group_name = 1; + */ + public Builder setGroupName( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupName_ = value; + onChanged(); + return this; + } + /** + * required string group_name = 1; + */ + public Builder clearGroupName() { + bitField0_ = (bitField0_ & ~0x00000001); + groupName_ = getDefaultInstance().getGroupName(); + onChanged(); + return this; + } + /** + * required string group_name = 1; + */ + public Builder setGroupNameBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupName_ = value; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:hbase.pb.CompleteRedistributeGroupRequest) + } + + static { + defaultInstance = new CompleteRedistributeGroupRequest(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:hbase.pb.CompleteRedistributeGroupRequest) + } + + public interface CompleteRedistributeGroupResponseOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required bool result = 1; + /** + * required bool result = 1; + */ + boolean hasResult(); + /** + * required bool result = 1; + */ + boolean getResult(); + } + /** + * Protobuf type {@code hbase.pb.CompleteRedistributeGroupResponse} + */ + public static final class CompleteRedistributeGroupResponse extends + com.google.protobuf.GeneratedMessage + implements CompleteRedistributeGroupResponseOrBuilder { + // Use CompleteRedistributeGroupResponse.newBuilder() to construct. + private CompleteRedistributeGroupResponse(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private CompleteRedistributeGroupResponse(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final CompleteRedistributeGroupResponse defaultInstance; + public static CompleteRedistributeGroupResponse getDefaultInstance() { + return defaultInstance; + } + + public CompleteRedistributeGroupResponse getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private CompleteRedistributeGroupResponse( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + result_ = input.readBool(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CompleteRedistributeGroupResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CompleteRedistributeGroupResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse.class, org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public CompleteRedistributeGroupResponse parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new CompleteRedistributeGroupResponse(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + private int bitField0_; + // required bool result = 1; + public static final int RESULT_FIELD_NUMBER = 1; + private boolean result_; + /** + * required bool result = 1; + */ + public boolean hasResult() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * required bool result = 1; + */ + public boolean getResult() { + return result_; + } + + private void initFields() { + result_ = false; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasResult()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBool(1, result_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(1, result_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse)) { + return super.equals(obj); + } + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse other = (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse) obj; + + boolean result = true; + result = result && (hasResult() == other.hasResult()); + if (hasResult()) { + result = result && (getResult() + == other.getResult()); + } + result = result && + getUnknownFields().equals(other.getUnknownFields()); + return result; + } + + private int memoizedHashCode = 0; + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptorForType().hashCode(); + if (hasResult()) { + hash = (37 * hash) + RESULT_FIELD_NUMBER; + hash = (53 * hash) + hashBoolean(getResult()); + } + hash = (29 * hash) + getUnknownFields().hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code hbase.pb.CompleteRedistributeGroupResponse} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponseOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CompleteRedistributeGroupResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CompleteRedistributeGroupResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse.class, org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse.Builder.class); + } + + // Construct using org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + result_ = false; + bitField0_ = (bitField0_ & ~0x00000001); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CompleteRedistributeGroupResponse_descriptor; + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse getDefaultInstanceForType() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse build() { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse buildPartial() { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse result = new org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.result_ = result_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse) { + return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse other) { + if (other == org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse.getDefaultInstance()) return this; + if (other.hasResult()) { + setResult(other.getResult()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasResult()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // required bool result = 1; + private boolean result_ ; + /** + * required bool result = 1; + */ + public boolean hasResult() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * required bool result = 1; + */ + public boolean getResult() { + return result_; + } + /** + * required bool result = 1; + */ + public Builder setResult(boolean value) { + bitField0_ |= 0x00000001; + result_ = value; + onChanged(); + return this; + } + /** + * required bool result = 1; + */ + public Builder clearResult() { + bitField0_ = (bitField0_ & ~0x00000001); + result_ = false; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:hbase.pb.CompleteRedistributeGroupResponse) + } + + static { + defaultInstance = new CompleteRedistributeGroupResponse(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:hbase.pb.CompleteRedistributeGroupResponse) + } + + public interface CheckGroupFavoredNodesRequestOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // required string group_name = 1; + /** + * required string group_name = 1; + */ + boolean hasGroupName(); + /** + * required string group_name = 1; + */ + java.lang.String getGroupName(); + /** + * required string group_name = 1; + */ + com.google.protobuf.ByteString + getGroupNameBytes(); + + // optional bool detailed = 2; + /** + * optional bool detailed = 2; + */ + boolean hasDetailed(); + /** + * optional bool detailed = 2; + */ + boolean getDetailed(); + } + /** + * Protobuf type {@code hbase.pb.CheckGroupFavoredNodesRequest} + */ + public static final class CheckGroupFavoredNodesRequest extends + com.google.protobuf.GeneratedMessage + implements CheckGroupFavoredNodesRequestOrBuilder { + // Use CheckGroupFavoredNodesRequest.newBuilder() to construct. + private CheckGroupFavoredNodesRequest(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private CheckGroupFavoredNodesRequest(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final CheckGroupFavoredNodesRequest defaultInstance; + public static CheckGroupFavoredNodesRequest getDefaultInstance() { + return defaultInstance; + } + + public CheckGroupFavoredNodesRequest getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private CheckGroupFavoredNodesRequest( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + groupName_ = input.readBytes(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + detailed_ = input.readBool(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CheckGroupFavoredNodesRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CheckGroupFavoredNodesRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest.class, org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public CheckGroupFavoredNodesRequest parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new CheckGroupFavoredNodesRequest(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + private int bitField0_; + // required string group_name = 1; + public static final int GROUP_NAME_FIELD_NUMBER = 1; + private java.lang.Object groupName_; + /** + * required string group_name = 1; + */ + public boolean hasGroupName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * required string group_name = 1; + */ + public java.lang.String getGroupName() { + java.lang.Object ref = groupName_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupName_ = s; + } + return s; + } + } + /** + * required string group_name = 1; + */ + public com.google.protobuf.ByteString + getGroupNameBytes() { + java.lang.Object ref = groupName_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + groupName_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + // optional bool detailed = 2; + public static final int DETAILED_FIELD_NUMBER = 2; + private boolean detailed_; + /** + * optional bool detailed = 2; + */ + public boolean hasDetailed() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + /** + * optional bool detailed = 2; + */ + public boolean getDetailed() { + return detailed_; + } + + private void initFields() { + groupName_ = ""; + detailed_ = false; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + if (!hasGroupName()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, getGroupNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBool(2, detailed_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBytesSize(1, getGroupNameBytes()); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(2, detailed_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest)) { + return super.equals(obj); + } + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest other = (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest) obj; + + boolean result = true; + result = result && (hasGroupName() == other.hasGroupName()); + if (hasGroupName()) { + result = result && getGroupName() + .equals(other.getGroupName()); + } + result = result && (hasDetailed() == other.hasDetailed()); + if (hasDetailed()) { + result = result && (getDetailed() + == other.getDetailed()); + } + result = result && + getUnknownFields().equals(other.getUnknownFields()); + return result; + } + + private int memoizedHashCode = 0; + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptorForType().hashCode(); + if (hasGroupName()) { + hash = (37 * hash) + GROUP_NAME_FIELD_NUMBER; + hash = (53 * hash) + getGroupName().hashCode(); + } + if (hasDetailed()) { + hash = (37 * hash) + DETAILED_FIELD_NUMBER; + hash = (53 * hash) + hashBoolean(getDetailed()); + } + hash = (29 * hash) + getUnknownFields().hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code hbase.pb.CheckGroupFavoredNodesRequest} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CheckGroupFavoredNodesRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CheckGroupFavoredNodesRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest.class, org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest.Builder.class); + } + + // Construct using org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + groupName_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + detailed_ = false; + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CheckGroupFavoredNodesRequest_descriptor; + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest getDefaultInstanceForType() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest build() { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest buildPartial() { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest result = new org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupName_ = groupName_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.detailed_ = detailed_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest) { + return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest other) { + if (other == org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest.getDefaultInstance()) return this; + if (other.hasGroupName()) { + bitField0_ |= 0x00000001; + groupName_ = other.groupName_; + onChanged(); + } + if (other.hasDetailed()) { + setDetailed(other.getDetailed()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupName()) { + + return false; + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // required string group_name = 1; + private java.lang.Object groupName_ = ""; + /** + * required string group_name = 1; + */ + public boolean hasGroupName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * required string group_name = 1; + */ + public java.lang.String getGroupName() { + java.lang.Object ref = groupName_; + if (!(ref instanceof java.lang.String)) { + java.lang.String s = ((com.google.protobuf.ByteString) ref) + .toStringUtf8(); + groupName_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + * required string group_name = 1; + */ + public com.google.protobuf.ByteString + getGroupNameBytes() { + java.lang.Object ref = groupName_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + groupName_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + * required string group_name = 1; + */ + public Builder setGroupName( + java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupName_ = value; + onChanged(); + return this; + } + /** + * required string group_name = 1; + */ + public Builder clearGroupName() { + bitField0_ = (bitField0_ & ~0x00000001); + groupName_ = getDefaultInstance().getGroupName(); + onChanged(); + return this; + } + /** + * required string group_name = 1; + */ + public Builder setGroupNameBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupName_ = value; + onChanged(); + return this; + } + + // optional bool detailed = 2; + private boolean detailed_ ; + /** + * optional bool detailed = 2; + */ + public boolean hasDetailed() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + /** + * optional bool detailed = 2; + */ + public boolean getDetailed() { + return detailed_; + } + /** + * optional bool detailed = 2; + */ + public Builder setDetailed(boolean value) { + bitField0_ |= 0x00000002; + detailed_ = value; + onChanged(); + return this; + } + /** + * optional bool detailed = 2; + */ + public Builder clearDetailed() { + bitField0_ = (bitField0_ & ~0x00000002); + detailed_ = false; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:hbase.pb.CheckGroupFavoredNodesRequest) + } + + static { + defaultInstance = new CheckGroupFavoredNodesRequest(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:hbase.pb.CheckGroupFavoredNodesRequest) + } + + public interface CheckGroupFavoredNodesResponseOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // repeated .hbase.pb.ServerName servers = 1; + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + java.util.List + getServersList(); + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName getServers(int index); + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + int getServersCount(); + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + java.util.List + getServersOrBuilderList(); + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder getServersOrBuilder( + int index); + } + /** + * Protobuf type {@code hbase.pb.CheckGroupFavoredNodesResponse} + */ + public static final class CheckGroupFavoredNodesResponse extends + com.google.protobuf.GeneratedMessage + implements CheckGroupFavoredNodesResponseOrBuilder { + // Use CheckGroupFavoredNodesResponse.newBuilder() to construct. + private CheckGroupFavoredNodesResponse(com.google.protobuf.GeneratedMessage.Builder builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private CheckGroupFavoredNodesResponse(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final CheckGroupFavoredNodesResponse defaultInstance; + public static CheckGroupFavoredNodesResponse getDefaultInstance() { + return defaultInstance; + } + + public CheckGroupFavoredNodesResponse getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private CheckGroupFavoredNodesResponse( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + servers_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000001; + } + servers_.add(input.readMessage(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.PARSER, extensionRegistry)); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + servers_ = java.util.Collections.unmodifiableList(servers_); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CheckGroupFavoredNodesResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CheckGroupFavoredNodesResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse.class, org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse.Builder.class); + } + + public static com.google.protobuf.Parser PARSER = + new com.google.protobuf.AbstractParser() { + public CheckGroupFavoredNodesResponse parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new CheckGroupFavoredNodesResponse(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + // repeated .hbase.pb.ServerName servers = 1; + public static final int SERVERS_FIELD_NUMBER = 1; + private java.util.List servers_; + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public java.util.List getServersList() { + return servers_; + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public java.util.List + getServersOrBuilderList() { + return servers_; + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public int getServersCount() { + return servers_.size(); + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName getServers(int index) { + return servers_.get(index); + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder getServersOrBuilder( + int index) { + return servers_.get(index); + } + + private void initFields() { + servers_ = java.util.Collections.emptyList(); + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + for (int i = 0; i < getServersCount(); i++) { + if (!getServers(i).isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + for (int i = 0; i < servers_.size(); i++) { + output.writeMessage(1, servers_.get(i)); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + for (int i = 0; i < servers_.size(); i++) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(1, servers_.get(i)); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse)) { + return super.equals(obj); + } + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse other = (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse) obj; + + boolean result = true; + result = result && getServersList() + .equals(other.getServersList()); + result = result && + getUnknownFields().equals(other.getUnknownFields()); + return result; + } + + private int memoizedHashCode = 0; + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptorForType().hashCode(); + if (getServersCount() > 0) { + hash = (37 * hash) + SERVERS_FIELD_NUMBER; + hash = (53 * hash) + getServersList().hashCode(); + } + hash = (29 * hash) + getUnknownFields().hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code hbase.pb.CheckGroupFavoredNodesResponse} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder + implements org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponseOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CheckGroupFavoredNodesResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CheckGroupFavoredNodesResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse.class, org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse.Builder.class); + } + + // Construct using org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + getServersFieldBuilder(); + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + if (serversBuilder_ == null) { + servers_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + } else { + serversBuilder_.clear(); + } + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.internal_static_hbase_pb_CheckGroupFavoredNodesResponse_descriptor; + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse getDefaultInstanceForType() { + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse build() { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse buildPartial() { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse result = new org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse(this); + int from_bitField0_ = bitField0_; + if (serversBuilder_ == null) { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + servers_ = java.util.Collections.unmodifiableList(servers_); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.servers_ = servers_; + } else { + result.servers_ = serversBuilder_.build(); + } + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse) { + return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse other) { + if (other == org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse.getDefaultInstance()) return this; + if (serversBuilder_ == null) { + if (!other.servers_.isEmpty()) { + if (servers_.isEmpty()) { + servers_ = other.servers_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureServersIsMutable(); + servers_.addAll(other.servers_); + } + onChanged(); + } + } else { + if (!other.servers_.isEmpty()) { + if (serversBuilder_.isEmpty()) { + serversBuilder_.dispose(); + serversBuilder_ = null; + servers_ = other.servers_; + bitField0_ = (bitField0_ & ~0x00000001); + serversBuilder_ = + com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ? + getServersFieldBuilder() : null; + } else { + serversBuilder_.addAllMessages(other.servers_); + } + } + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + for (int i = 0; i < getServersCount(); i++) { + if (!getServers(i).isInitialized()) { + + return false; + } + } + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // repeated .hbase.pb.ServerName servers = 1; + private java.util.List servers_ = + java.util.Collections.emptyList(); + private void ensureServersIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + servers_ = new java.util.ArrayList(servers_); + bitField0_ |= 0x00000001; + } + } + + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder> serversBuilder_; + + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public java.util.List getServersList() { + if (serversBuilder_ == null) { + return java.util.Collections.unmodifiableList(servers_); + } else { + return serversBuilder_.getMessageList(); + } + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public int getServersCount() { + if (serversBuilder_ == null) { + return servers_.size(); + } else { + return serversBuilder_.getCount(); + } + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName getServers(int index) { + if (serversBuilder_ == null) { + return servers_.get(index); + } else { + return serversBuilder_.getMessage(index); + } + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public Builder setServers( + int index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName value) { + if (serversBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureServersIsMutable(); + servers_.set(index, value); + onChanged(); + } else { + serversBuilder_.setMessage(index, value); + } + return this; + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public Builder setServers( + int index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.Builder builderForValue) { + if (serversBuilder_ == null) { + ensureServersIsMutable(); + servers_.set(index, builderForValue.build()); + onChanged(); + } else { + serversBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public Builder addServers(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName value) { + if (serversBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureServersIsMutable(); + servers_.add(value); + onChanged(); + } else { + serversBuilder_.addMessage(value); + } + return this; + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public Builder addServers( + int index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName value) { + if (serversBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureServersIsMutable(); + servers_.add(index, value); + onChanged(); + } else { + serversBuilder_.addMessage(index, value); + } + return this; + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public Builder addServers( + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.Builder builderForValue) { + if (serversBuilder_ == null) { + ensureServersIsMutable(); + servers_.add(builderForValue.build()); + onChanged(); + } else { + serversBuilder_.addMessage(builderForValue.build()); + } + return this; + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public Builder addServers( + int index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.Builder builderForValue) { + if (serversBuilder_ == null) { + ensureServersIsMutable(); + servers_.add(index, builderForValue.build()); + onChanged(); + } else { + serversBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public Builder addAllServers( + java.lang.Iterable values) { + if (serversBuilder_ == null) { + ensureServersIsMutable(); + super.addAll(values, servers_); + onChanged(); + } else { + serversBuilder_.addAllMessages(values); + } + return this; + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public Builder clearServers() { + if (serversBuilder_ == null) { + servers_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + } else { + serversBuilder_.clear(); + } + return this; + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public Builder removeServers(int index) { + if (serversBuilder_ == null) { + ensureServersIsMutable(); + servers_.remove(index); + onChanged(); + } else { + serversBuilder_.remove(index); + } + return this; + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.Builder getServersBuilder( + int index) { + return getServersFieldBuilder().getBuilder(index); + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder getServersOrBuilder( + int index) { + if (serversBuilder_ == null) { + return servers_.get(index); } else { + return serversBuilder_.getMessageOrBuilder(index); + } + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public java.util.List + getServersOrBuilderList() { + if (serversBuilder_ != null) { + return serversBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(servers_); + } + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.Builder addServersBuilder() { + return getServersFieldBuilder().addBuilder( + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.getDefaultInstance()); + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.Builder addServersBuilder( + int index) { + return getServersFieldBuilder().addBuilder( + index, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.getDefaultInstance()); + } + /** + * repeated .hbase.pb.ServerName servers = 1; + */ + public java.util.List + getServersBuilderList() { + return getServersFieldBuilder().getBuilderList(); + } + private com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder> + getServersFieldBuilder() { + if (serversBuilder_ == null) { + serversBuilder_ = new com.google.protobuf.RepeatedFieldBuilder< + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.Builder, org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder>( + servers_, + ((bitField0_ & 0x00000001) == 0x00000001), + getParentForChildren(), + isClean()); + servers_ = null; + } + return serversBuilder_; + } + + // @@protoc_insertion_point(builder_scope:hbase.pb.CheckGroupFavoredNodesResponse) + } + + static { + defaultInstance = new CheckGroupFavoredNodesResponse(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:hbase.pb.CheckGroupFavoredNodesResponse) + } + /** * Protobuf service {@code hbase.pb.RSGroupAdminService} */ @@ -10834,6 +14080,30 @@ public final class RSGroupAdminProtos { org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosRequest request, com.google.protobuf.RpcCallback done); + /** + * rpc RedistributeGroup(.hbase.pb.RedistributeGroupRequest) returns (.hbase.pb.RedistributeGroupResponse); + */ + public abstract void redistributeGroup( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest request, + com.google.protobuf.RpcCallback done); + + /** + * rpc CompleteRedistributeGroup(.hbase.pb.CompleteRedistributeGroupRequest) returns (.hbase.pb.CompleteRedistributeGroupResponse); + */ + public abstract void completeRedistributeGroup( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest request, + com.google.protobuf.RpcCallback done); + + /** + * rpc checkGroupFavoredNodes(.hbase.pb.CheckGroupFavoredNodesRequest) returns (.hbase.pb.CheckGroupFavoredNodesResponse); + */ + public abstract void checkGroupFavoredNodes( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest request, + com.google.protobuf.RpcCallback done); + } public static com.google.protobuf.Service newReflectiveService( @@ -10911,6 +14181,30 @@ public final class RSGroupAdminProtos { impl.listRSGroupInfos(controller, request, done); } + @java.lang.Override + public void redistributeGroup( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest request, + com.google.protobuf.RpcCallback done) { + impl.redistributeGroup(controller, request, done); + } + + @java.lang.Override + public void completeRedistributeGroup( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest request, + com.google.protobuf.RpcCallback done) { + impl.completeRedistributeGroup(controller, request, done); + } + + @java.lang.Override + public void checkGroupFavoredNodes( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest request, + com.google.protobuf.RpcCallback done) { + impl.checkGroupFavoredNodes(controller, request, done); + } + }; } @@ -10951,6 +14245,12 @@ public final class RSGroupAdminProtos { return impl.balanceRSGroup(controller, (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupRequest)request); case 8: return impl.listRSGroupInfos(controller, (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosRequest)request); + case 9: + return impl.redistributeGroup(controller, (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest)request); + case 10: + return impl.completeRedistributeGroup(controller, (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest)request); + case 11: + return impl.checkGroupFavoredNodes(controller, (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest)request); default: throw new java.lang.AssertionError("Can't get here."); } @@ -10983,6 +14283,12 @@ public final class RSGroupAdminProtos { return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupRequest.getDefaultInstance(); case 8: return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosRequest.getDefaultInstance(); + case 9: + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest.getDefaultInstance(); + case 10: + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest.getDefaultInstance(); + case 11: + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest.getDefaultInstance(); default: throw new java.lang.AssertionError("Can't get here."); } @@ -11015,6 +14321,12 @@ public final class RSGroupAdminProtos { return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupResponse.getDefaultInstance(); case 8: return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosResponse.getDefaultInstance(); + case 9: + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse.getDefaultInstance(); + case 10: + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse.getDefaultInstance(); + case 11: + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse.getDefaultInstance(); default: throw new java.lang.AssertionError("Can't get here."); } @@ -11095,6 +14407,30 @@ public final class RSGroupAdminProtos { org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosRequest request, com.google.protobuf.RpcCallback done); + /** + * rpc RedistributeGroup(.hbase.pb.RedistributeGroupRequest) returns (.hbase.pb.RedistributeGroupResponse); + */ + public abstract void redistributeGroup( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest request, + com.google.protobuf.RpcCallback done); + + /** + * rpc CompleteRedistributeGroup(.hbase.pb.CompleteRedistributeGroupRequest) returns (.hbase.pb.CompleteRedistributeGroupResponse); + */ + public abstract void completeRedistributeGroup( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest request, + com.google.protobuf.RpcCallback done); + + /** + * rpc checkGroupFavoredNodes(.hbase.pb.CheckGroupFavoredNodesRequest) returns (.hbase.pb.CheckGroupFavoredNodesResponse); + */ + public abstract void checkGroupFavoredNodes( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest request, + com.google.protobuf.RpcCallback done); + public static final com.google.protobuf.Descriptors.ServiceDescriptor getDescriptor() { @@ -11162,6 +14498,21 @@ public final class RSGroupAdminProtos { com.google.protobuf.RpcUtil.specializeCallback( done)); return; + case 9: + this.redistributeGroup(controller, (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest)request, + com.google.protobuf.RpcUtil.specializeCallback( + done)); + return; + case 10: + this.completeRedistributeGroup(controller, (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest)request, + com.google.protobuf.RpcUtil.specializeCallback( + done)); + return; + case 11: + this.checkGroupFavoredNodes(controller, (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest)request, + com.google.protobuf.RpcUtil.specializeCallback( + done)); + return; default: throw new java.lang.AssertionError("Can't get here."); } @@ -11194,6 +14545,12 @@ public final class RSGroupAdminProtos { return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupRequest.getDefaultInstance(); case 8: return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosRequest.getDefaultInstance(); + case 9: + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest.getDefaultInstance(); + case 10: + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest.getDefaultInstance(); + case 11: + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest.getDefaultInstance(); default: throw new java.lang.AssertionError("Can't get here."); } @@ -11226,6 +14583,12 @@ public final class RSGroupAdminProtos { return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.BalanceRSGroupResponse.getDefaultInstance(); case 8: return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosResponse.getDefaultInstance(); + case 9: + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse.getDefaultInstance(); + case 10: + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse.getDefaultInstance(); + case 11: + return org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse.getDefaultInstance(); default: throw new java.lang.AssertionError("Can't get here."); } @@ -11381,6 +14744,51 @@ public final class RSGroupAdminProtos { org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosResponse.class, org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosResponse.getDefaultInstance())); } + + public void redistributeGroup( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest request, + com.google.protobuf.RpcCallback done) { + channel.callMethod( + getDescriptor().getMethods().get(9), + controller, + request, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse.getDefaultInstance(), + com.google.protobuf.RpcUtil.generalizeCallback( + done, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse.class, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse.getDefaultInstance())); + } + + public void completeRedistributeGroup( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest request, + com.google.protobuf.RpcCallback done) { + channel.callMethod( + getDescriptor().getMethods().get(10), + controller, + request, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse.getDefaultInstance(), + com.google.protobuf.RpcUtil.generalizeCallback( + done, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse.class, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse.getDefaultInstance())); + } + + public void checkGroupFavoredNodes( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest request, + com.google.protobuf.RpcCallback done) { + channel.callMethod( + getDescriptor().getMethods().get(11), + controller, + request, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse.getDefaultInstance(), + com.google.protobuf.RpcUtil.generalizeCallback( + done, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse.class, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse.getDefaultInstance())); + } } public static BlockingInterface newBlockingStub( @@ -11433,6 +14841,21 @@ public final class RSGroupAdminProtos { com.google.protobuf.RpcController controller, org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosRequest request) throws com.google.protobuf.ServiceException; + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse redistributeGroup( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest request) + throws com.google.protobuf.ServiceException; + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse completeRedistributeGroup( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest request) + throws com.google.protobuf.ServiceException; + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse checkGroupFavoredNodes( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest request) + throws com.google.protobuf.ServiceException; } private static final class BlockingStub implements BlockingInterface { @@ -11549,6 +14972,42 @@ public final class RSGroupAdminProtos { org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.ListRSGroupInfosResponse.getDefaultInstance()); } + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse redistributeGroup( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest request) + throws com.google.protobuf.ServiceException { + return (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse) channel.callBlockingMethod( + getDescriptor().getMethods().get(9), + controller, + request, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupResponse.getDefaultInstance()); + } + + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse completeRedistributeGroup( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest request) + throws com.google.protobuf.ServiceException { + return (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse) channel.callBlockingMethod( + getDescriptor().getMethods().get(10), + controller, + request, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupResponse.getDefaultInstance()); + } + + + public org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse checkGroupFavoredNodes( + com.google.protobuf.RpcController controller, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest request) + throws com.google.protobuf.ServiceException { + return (org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse) channel.callBlockingMethod( + getDescriptor().getMethods().get(11), + controller, + request, + org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse.getDefaultInstance()); + } + } // @@protoc_insertion_point(class_scope:hbase.pb.RSGroupAdminService) @@ -11654,6 +15113,36 @@ public final class RSGroupAdminProtos { private static com.google.protobuf.GeneratedMessage.FieldAccessorTable internal_static_hbase_pb_GetRSGroupInfoOfServerResponse_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_hbase_pb_RedistributeGroupRequest_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_hbase_pb_RedistributeGroupRequest_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_hbase_pb_RedistributeGroupResponse_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_hbase_pb_RedistributeGroupResponse_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_hbase_pb_CompleteRedistributeGroupRequest_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_hbase_pb_CompleteRedistributeGroupRequest_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_hbase_pb_CompleteRedistributeGroupResponse_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_hbase_pb_CompleteRedistributeGroupResponse_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_hbase_pb_CheckGroupFavoredNodesRequest_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_hbase_pb_CheckGroupFavoredNodesRequest_fieldAccessorTable; + private static com.google.protobuf.Descriptors.Descriptor + internal_static_hbase_pb_CheckGroupFavoredNodesResponse_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_hbase_pb_CheckGroupFavoredNodesResponse_fieldAccessorTable; public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { @@ -11692,28 +15181,45 @@ public final class RSGroupAdminProtos { "t\022$\n\006server\030\002 \002(\0132\024.hbase.pb.ServerName\"" + "O\n\036GetRSGroupInfoOfServerResponse\022-\n\016r_s" + "_group_info\030\001 \001(\0132\025.hbase.pb.RSGroupInfo" + - "2\241\006\n\023RSGroupAdminService\022S\n\016GetRSGroupIn", - "fo\022\037.hbase.pb.GetRSGroupInfoRequest\032 .hb" + - "ase.pb.GetRSGroupInfoResponse\022h\n\025GetRSGr" + - "oupInfoOfTable\022&.hbase.pb.GetRSGroupInfo" + - "OfTableRequest\032\'.hbase.pb.GetRSGroupInfo" + - "OfTableResponse\022k\n\026GetRSGroupInfoOfServe" + - "r\022\'.hbase.pb.GetRSGroupInfoOfServerReque" + - "st\032(.hbase.pb.GetRSGroupInfoOfServerResp" + - "onse\022J\n\013MoveServers\022\034.hbase.pb.MoveServe" + - "rsRequest\032\035.hbase.pb.MoveServersResponse" + - "\022G\n\nMoveTables\022\033.hbase.pb.MoveTablesRequ", - "est\032\034.hbase.pb.MoveTablesResponse\022G\n\nAdd" + - "RSGroup\022\033.hbase.pb.AddRSGroupRequest\032\034.h" + - "base.pb.AddRSGroupResponse\022P\n\rRemoveRSGr" + - "oup\022\036.hbase.pb.RemoveRSGroupRequest\032\037.hb" + - "ase.pb.RemoveRSGroupResponse\022S\n\016BalanceR" + - "SGroup\022\037.hbase.pb.BalanceRSGroupRequest\032" + - " .hbase.pb.BalanceRSGroupResponse\022Y\n\020Lis" + - "tRSGroupInfos\022!.hbase.pb.ListRSGroupInfo" + - "sRequest\032\".hbase.pb.ListRSGroupInfosResp" + - "onseBH\n*org.apache.hadoop.hbase.protobuf", - ".generatedB\022RSGroupAdminProtosH\001\210\001\001\240\001\001" + "\".\n\030RedistributeGroupRequest\022\022\n\ngroup_na", + "me\030\001 \002(\t\"+\n\031RedistributeGroupResponse\022\016\n" + + "\006result\030\001 \002(\010\"6\n CompleteRedistributeGro" + + "upRequest\022\022\n\ngroup_name\030\001 \002(\t\"3\n!Complet" + + "eRedistributeGroupResponse\022\016\n\006result\030\001 \002" + + "(\010\"E\n\035CheckGroupFavoredNodesRequest\022\022\n\ng" + + "roup_name\030\001 \002(\t\022\020\n\010detailed\030\002 \001(\010\"G\n\036Che" + + "ckGroupFavoredNodesResponse\022%\n\007servers\030\001" + + " \003(\0132\024.hbase.pb.ServerName2\342\010\n\023RSGroupAd" + + "minService\022S\n\016GetRSGroupInfo\022\037.hbase.pb." + + "GetRSGroupInfoRequest\032 .hbase.pb.GetRSGr", + "oupInfoResponse\022h\n\025GetRSGroupInfoOfTable" + + "\022&.hbase.pb.GetRSGroupInfoOfTableRequest" + + "\032\'.hbase.pb.GetRSGroupInfoOfTableRespons" + + "e\022k\n\026GetRSGroupInfoOfServer\022\'.hbase.pb.G" + + "etRSGroupInfoOfServerRequest\032(.hbase.pb." + + "GetRSGroupInfoOfServerResponse\022J\n\013MoveSe" + + "rvers\022\034.hbase.pb.MoveServersRequest\032\035.hb" + + "ase.pb.MoveServersResponse\022G\n\nMoveTables" + + "\022\033.hbase.pb.MoveTablesRequest\032\034.hbase.pb" + + ".MoveTablesResponse\022G\n\nAddRSGroup\022\033.hbas", + "e.pb.AddRSGroupRequest\032\034.hbase.pb.AddRSG" + + "roupResponse\022P\n\rRemoveRSGroup\022\036.hbase.pb" + + ".RemoveRSGroupRequest\032\037.hbase.pb.RemoveR" + + "SGroupResponse\022S\n\016BalanceRSGroup\022\037.hbase" + + ".pb.BalanceRSGroupRequest\032 .hbase.pb.Bal" + + "anceRSGroupResponse\022Y\n\020ListRSGroupInfos\022" + + "!.hbase.pb.ListRSGroupInfosRequest\032\".hba" + + "se.pb.ListRSGroupInfosResponse\022\\\n\021Redist" + + "ributeGroup\022\".hbase.pb.RedistributeGroup" + + "Request\032#.hbase.pb.RedistributeGroupResp", + "onse\022t\n\031CompleteRedistributeGroup\022*.hbas" + + "e.pb.CompleteRedistributeGroupRequest\032+." + + "hbase.pb.CompleteRedistributeGroupRespon" + + "se\022k\n\026checkGroupFavoredNodes\022\'.hbase.pb." + + "CheckGroupFavoredNodesRequest\032(.hbase.pb" + + ".CheckGroupFavoredNodesResponseBH\n*org.a" + + "pache.hadoop.hbase.protobuf.generatedB\022R" + + "SGroupAdminProtosH\001\210\001\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -11840,6 +15346,42 @@ public final class RSGroupAdminProtos { com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_hbase_pb_GetRSGroupInfoOfServerResponse_descriptor, new java.lang.String[] { "RSGroupInfo", }); + internal_static_hbase_pb_RedistributeGroupRequest_descriptor = + getDescriptor().getMessageTypes().get(20); + internal_static_hbase_pb_RedistributeGroupRequest_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_hbase_pb_RedistributeGroupRequest_descriptor, + new java.lang.String[] { "GroupName", }); + internal_static_hbase_pb_RedistributeGroupResponse_descriptor = + getDescriptor().getMessageTypes().get(21); + internal_static_hbase_pb_RedistributeGroupResponse_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_hbase_pb_RedistributeGroupResponse_descriptor, + new java.lang.String[] { "Result", }); + internal_static_hbase_pb_CompleteRedistributeGroupRequest_descriptor = + getDescriptor().getMessageTypes().get(22); + internal_static_hbase_pb_CompleteRedistributeGroupRequest_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_hbase_pb_CompleteRedistributeGroupRequest_descriptor, + new java.lang.String[] { "GroupName", }); + internal_static_hbase_pb_CompleteRedistributeGroupResponse_descriptor = + getDescriptor().getMessageTypes().get(23); + internal_static_hbase_pb_CompleteRedistributeGroupResponse_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_hbase_pb_CompleteRedistributeGroupResponse_descriptor, + new java.lang.String[] { "Result", }); + internal_static_hbase_pb_CheckGroupFavoredNodesRequest_descriptor = + getDescriptor().getMessageTypes().get(24); + internal_static_hbase_pb_CheckGroupFavoredNodesRequest_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_hbase_pb_CheckGroupFavoredNodesRequest_descriptor, + new java.lang.String[] { "GroupName", "Detailed", }); + internal_static_hbase_pb_CheckGroupFavoredNodesResponse_descriptor = + getDescriptor().getMessageTypes().get(25); + internal_static_hbase_pb_CheckGroupFavoredNodesResponse_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_hbase_pb_CheckGroupFavoredNodesResponse_descriptor, + new java.lang.String[] { "Servers", }); return null; } }; diff --git a/hbase-protocol/src/main/protobuf/RSGroupAdmin.proto b/hbase-protocol/src/main/protobuf/RSGroupAdmin.proto index fda9b09..b584810 100644 --- a/hbase-protocol/src/main/protobuf/RSGroupAdmin.proto +++ b/hbase-protocol/src/main/protobuf/RSGroupAdmin.proto @@ -106,6 +106,32 @@ message GetRSGroupInfoOfServerResponse { optional RSGroupInfo r_s_group_info = 1; } +message RedistributeGroupRequest { + required string group_name = 1; +} + +message RedistributeGroupResponse { + required bool result = 1; +} + +message CompleteRedistributeGroupRequest { + required string group_name = 1; +} + +message CompleteRedistributeGroupResponse { + required bool result = 1; +} + +message CheckGroupFavoredNodesRequest { + required string group_name = 1; + optional bool detailed = 2; +} + +message CheckGroupFavoredNodesResponse { + repeated ServerName servers = 1; +} + + service RSGroupAdminService { rpc GetRSGroupInfo(GetRSGroupInfoRequest) returns (GetRSGroupInfoResponse); @@ -133,4 +159,13 @@ service RSGroupAdminService { rpc ListRSGroupInfos(ListRSGroupInfosRequest) returns (ListRSGroupInfosResponse); + + rpc RedistributeGroup(RedistributeGroupRequest) + returns (RedistributeGroupResponse); + + rpc CompleteRedistributeGroup(CompleteRedistributeGroupRequest) + returns (CompleteRedistributeGroupResponse); + + rpc checkGroupFavoredNodes(CheckGroupFavoredNodesRequest) + returns (CheckGroupFavoredNodesResponse); } diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/AssignmentVerificationReport.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/AssignmentVerificationReport.java new file mode 100644 index 0000000..858394c --- /dev/null +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/AssignmentVerificationReport.java @@ -0,0 +1,641 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rsgroup; + +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.master.SnapshotOfRegionAssignmentFromMeta; +import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper; +import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan; +/** + * Helper class that is used by {@link RegionPlacementMaintainer} to print + * information for favored nodes + * + */ +@InterfaceAudience.Private +public class AssignmentVerificationReport { + private static final Log LOG = LogFactory.getLog( + AssignmentVerificationReport.class.getName()); + + private TableName tableName = null; + private boolean enforceLocality = false; + private boolean isFilledUp = false; + + private int totalRegions = 0; + private int totalRegionServers = 0; + // for unassigned regions + private List unAssignedRegionsList = + new ArrayList(); + + // For regions without valid favored nodes + private List regionsWithoutValidFavoredNodes = + new ArrayList(); + + // For regions not running on the favored nodes + private List nonFavoredAssignedRegionList = + new ArrayList(); + + // For regions running on the favored nodes + private int totalFavoredAssignments = 0; + private int[] favoredNodes = new int[FavoredNodeAssignmentHelper.FAVORED_NODES_NUM]; + private float[] favoredNodesLocalitySummary = + new float[FavoredNodeAssignmentHelper.FAVORED_NODES_NUM]; + private float actualLocalitySummary = 0; + + // For region balancing information + private float avgRegionsOnRS = 0; + private int maxRegionsOnRS = 0; + private int minRegionsOnRS = Integer.MAX_VALUE; + private Set mostLoadedRSSet = + new HashSet(); + private Set leastLoadedRSSet = + new HashSet(); + + private float avgDispersionScore = 0; + private float maxDispersionScore = 0; + private Set maxDispersionScoreServerSet = + new HashSet(); + private float minDispersionScore = Float.MAX_VALUE; + private Set minDispersionScoreServerSet = + new HashSet(); + + private float avgDispersionNum = 0; + private float maxDispersionNum = 0; + private Set maxDispersionNumServerSet = + new HashSet(); + private float minDispersionNum = Float.MAX_VALUE; + private Set minDispersionNumServerSet = + new HashSet(); + + public void fillUp(TableName tableName, SnapshotOfRegionAssignmentFromMeta snapshot, + Map> regionLocalityMap) { + // Set the table name + this.tableName = tableName; + + // Get all the regions for this table + List regionInfoList = + snapshot.getTableToRegionMap().get(tableName); + // Get the total region num for the current table + this.totalRegions = regionInfoList.size(); + + // Get the existing assignment plan + FavoredNodesPlan favoredNodesAssignment = snapshot.getExistingAssignmentPlan(); + // Get the region to region server mapping + Map currentAssignment = + snapshot.getRegionToRegionServerMap(); + // Initialize the server to its hosing region counter map + Map serverToHostingRegionCounterMap = + new HashMap(); + + Map primaryRSToRegionCounterMap = + new HashMap(); + Map> primaryToSecTerRSMap = + new HashMap>(); + + // Check the favored nodes and its locality information + // Also keep tracker of the most loaded and least loaded region servers + for (HRegionInfo region : regionInfoList) { + try { + ServerName currentRS = currentAssignment.get(region); + // Handle unassigned regions + if (currentRS == null) { + unAssignedRegionsList.add(region); + continue; + } + + // Keep updating the server to is hosting region counter map + Integer hostRegionCounter = serverToHostingRegionCounterMap.get(currentRS); + if (hostRegionCounter == null) { + hostRegionCounter = Integer.valueOf(0); + } + hostRegionCounter = hostRegionCounter.intValue() + 1; + serverToHostingRegionCounterMap.put(currentRS, hostRegionCounter); + + // Get the favored nodes from the assignment plan and verify it. + List favoredNodes = favoredNodesAssignment.getFavoredNodes(region); + if (favoredNodes == null || + favoredNodes.size() != FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) { + regionsWithoutValidFavoredNodes.add(region); + continue; + } + // Get the primary, secondary and tertiary region server + ServerName primaryRS = + favoredNodes.get(FavoredNodesPlan.Position.PRIMARY.ordinal()); + ServerName secondaryRS = + favoredNodes.get(FavoredNodesPlan.Position.SECONDARY.ordinal()); + ServerName tertiaryRS = + favoredNodes.get(FavoredNodesPlan.Position.TERTIARY.ordinal()); + + // Update the primary rs to its region set map + Integer regionCounter = primaryRSToRegionCounterMap.get(primaryRS); + if (regionCounter == null) { + regionCounter = Integer.valueOf(0); + } + regionCounter = regionCounter.intValue() + 1; + primaryRSToRegionCounterMap.put(primaryRS, regionCounter); + + // Update the primary rs to secondary and tertiary rs map + Set secAndTerSet = primaryToSecTerRSMap.get(primaryRS); + if (secAndTerSet == null) { + secAndTerSet = new HashSet(); + } + secAndTerSet.add(secondaryRS); + secAndTerSet.add(tertiaryRS); + primaryToSecTerRSMap.put(primaryRS, secAndTerSet); + + // Get the position of the current region server in the favored nodes list + FavoredNodesPlan.Position favoredNodePosition = + FavoredNodesPlan.getFavoredServerPosition(favoredNodes, currentRS); + + // Handle the non favored assignment. + if (favoredNodePosition == null) { + nonFavoredAssignedRegionList.add(region); + continue; + } + // Increase the favored nodes assignment. + this.favoredNodes[favoredNodePosition.ordinal()]++; + totalFavoredAssignments++; + + // Summary the locality information for each favored nodes + if (regionLocalityMap != null) { + // Set the enforce locality as true; + this.enforceLocality = true; + + // Get the region degree locality map + Map regionDegreeLocalityMap = + regionLocalityMap.get(region.getEncodedName()); + if (regionDegreeLocalityMap == null) { + continue; // ignore the region which doesn't have any store files. + } + + // Get the locality summary for each favored nodes + for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) { + ServerName favoredNode = favoredNodes.get(p.ordinal()); + // Get the locality for the current favored nodes + Float locality = + regionDegreeLocalityMap.get(favoredNode.getHostname()); + if (locality != null) { + this.favoredNodesLocalitySummary[p.ordinal()] += locality; + } + } + + // Get the locality summary for the current region server + Float actualLocality = + regionDegreeLocalityMap.get(currentRS.getHostname()); + if (actualLocality != null) { + this.actualLocalitySummary += actualLocality; + } + } + } catch (Exception e) { + LOG.error("Cannot verify the region assignment for region " + + ((region == null) ? " null " : region.getRegionNameAsString()) + + "because of " + e); + } + } + + float dispersionScoreSummary = 0; + float dispersionNumSummary = 0; + // Calculate the secondary score for each primary region server + for (Map.Entry entry : + primaryRSToRegionCounterMap.entrySet()) { + ServerName primaryRS = entry.getKey(); + Integer regionsOnPrimary = entry.getValue(); + + // Process the dispersion number and score + float dispersionScore = 0; + int dispersionNum = 0; + if (primaryToSecTerRSMap.get(primaryRS) != null + && regionsOnPrimary.intValue() != 0) { + dispersionNum = primaryToSecTerRSMap.get(primaryRS).size(); + dispersionScore = dispersionNum / + ((float) regionsOnPrimary.intValue() * 2); + } + // Update the max dispersion score + if (dispersionScore > this.maxDispersionScore) { + this.maxDispersionScoreServerSet.clear(); + this.maxDispersionScoreServerSet.add(primaryRS); + this.maxDispersionScore = dispersionScore; + } else if (dispersionScore == this.maxDispersionScore) { + this.maxDispersionScoreServerSet.add(primaryRS); + } + + // Update the max dispersion num + if (dispersionNum > this.maxDispersionNum) { + this.maxDispersionNumServerSet.clear(); + this.maxDispersionNumServerSet.add(primaryRS); + this.maxDispersionNum = dispersionNum; + } else if (dispersionNum == this.maxDispersionNum) { + this.maxDispersionNumServerSet.add(primaryRS); + } + + // Update the min dispersion score + if (dispersionScore < this.minDispersionScore) { + this.minDispersionScoreServerSet.clear(); + this.minDispersionScoreServerSet.add(primaryRS); + this.minDispersionScore = dispersionScore; + } else if (dispersionScore == this.minDispersionScore) { + this.minDispersionScoreServerSet.add(primaryRS); + } + + // Update the min dispersion num + if (dispersionNum < this.minDispersionNum) { + this.minDispersionNumServerSet.clear(); + this.minDispersionNumServerSet.add(primaryRS); + this.minDispersionNum = dispersionNum; + } else if (dispersionNum == this.minDispersionNum) { + this.minDispersionNumServerSet.add(primaryRS); + } + + dispersionScoreSummary += dispersionScore; + dispersionNumSummary += dispersionNum; + } + + // Update the avg dispersion score + if (primaryRSToRegionCounterMap.keySet().size() != 0) { + this.avgDispersionScore = dispersionScoreSummary / + (float) primaryRSToRegionCounterMap.keySet().size(); + this.avgDispersionNum = dispersionNumSummary / + (float) primaryRSToRegionCounterMap.keySet().size(); + } + + // Fill up the most loaded and least loaded region server information + for (Map.Entry entry : + serverToHostingRegionCounterMap.entrySet()) { + ServerName currentRS = entry.getKey(); + int hostRegionCounter = entry.getValue().intValue(); + + // Update the most loaded region server list and maxRegionsOnRS + if (hostRegionCounter > this.maxRegionsOnRS) { + maxRegionsOnRS = hostRegionCounter; + this.mostLoadedRSSet.clear(); + this.mostLoadedRSSet.add(currentRS); + } else if (hostRegionCounter == this.maxRegionsOnRS) { + this.mostLoadedRSSet.add(currentRS); + } + + // Update the least loaded region server list and minRegionsOnRS + if (hostRegionCounter < this.minRegionsOnRS) { + this.minRegionsOnRS = hostRegionCounter; + this.leastLoadedRSSet.clear(); + this.leastLoadedRSSet.add(currentRS); + } else if (hostRegionCounter == this.minRegionsOnRS) { + this.leastLoadedRSSet.add(currentRS); + } + } + + // and total region servers + this.totalRegionServers = serverToHostingRegionCounterMap.keySet().size(); + this.avgRegionsOnRS = (totalRegionServers == 0) ? 0 : + (totalRegions / (float) totalRegionServers); + // Set the isFilledUp as true + isFilledUp = true; + } + + /** + * Use this to project the dispersion scores + * @param tableName + * @param snapshot + * @param newPlan + */ + public void fillUpDispersion(TableName tableName, + SnapshotOfRegionAssignmentFromMeta snapshot, FavoredNodesPlan newPlan) { + // Set the table name + this.tableName = tableName; + // Get all the regions for this table + List regionInfoList = snapshot.getTableToRegionMap().get( + tableName); + // Get the total region num for the current table + this.totalRegions = regionInfoList.size(); + FavoredNodesPlan plan = null; + if (newPlan == null) { + plan = snapshot.getExistingAssignmentPlan(); + } else { + plan = newPlan; + } + // Get the region to region server mapping + Map primaryRSToRegionCounterMap = + new HashMap(); + Map> primaryToSecTerRSMap = + new HashMap>(); + + // Check the favored nodes and its locality information + // Also keep tracker of the most loaded and least loaded region servers + for (HRegionInfo region : regionInfoList) { + try { + // Get the favored nodes from the assignment plan and verify it. + List favoredNodes = plan.getFavoredNodes(region); + if (favoredNodes == null + || favoredNodes.size() != FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) { + regionsWithoutValidFavoredNodes.add(region); + continue; + } + // Get the primary, secondary and tertiary region server + ServerName primaryRS = favoredNodes + .get(FavoredNodesPlan.Position.PRIMARY.ordinal()); + ServerName secondaryRS = favoredNodes + .get(FavoredNodesPlan.Position.SECONDARY.ordinal()); + ServerName tertiaryRS = favoredNodes + .get(FavoredNodesPlan.Position.TERTIARY.ordinal()); + + // Update the primary rs to its region set map + Integer regionCounter = primaryRSToRegionCounterMap.get(primaryRS); + if (regionCounter == null) { + regionCounter = Integer.valueOf(0); + } + regionCounter = regionCounter.intValue() + 1; + primaryRSToRegionCounterMap.put(primaryRS, regionCounter); + + // Update the primary rs to secondary and tertiary rs map + Set secAndTerSet = primaryToSecTerRSMap.get(primaryRS); + if (secAndTerSet == null) { + secAndTerSet = new HashSet(); + } + secAndTerSet.add(secondaryRS); + secAndTerSet.add(tertiaryRS); + primaryToSecTerRSMap.put(primaryRS, secAndTerSet); + } catch (Exception e) { + LOG.error("Cannot verify the region assignment for region " + + ((region == null) ? " null " : region.getRegionNameAsString()) + + "because of " + e); + } + } + float dispersionScoreSummary = 0; + float dispersionNumSummary = 0; + // Calculate the secondary score for each primary region server + for (Map.Entry entry : + primaryRSToRegionCounterMap.entrySet()) { + ServerName primaryRS = entry.getKey(); + Integer regionsOnPrimary = entry.getValue(); + + // Process the dispersion number and score + float dispersionScore = 0; + int dispersionNum = 0; + if (primaryToSecTerRSMap.get(primaryRS) != null + && regionsOnPrimary.intValue() != 0) { + dispersionNum = primaryToSecTerRSMap.get(primaryRS).size(); + dispersionScore = dispersionNum / + ((float) regionsOnPrimary.intValue() * 2); + } + + // Update the max dispersion num + if (dispersionNum > this.maxDispersionNum) { + this.maxDispersionNumServerSet.clear(); + this.maxDispersionNumServerSet.add(primaryRS); + this.maxDispersionNum = dispersionNum; + } else if (dispersionNum == this.maxDispersionNum) { + this.maxDispersionNumServerSet.add(primaryRS); + } + + // Update the min dispersion score + if (dispersionScore < this.minDispersionScore) { + this.minDispersionScoreServerSet.clear(); + this.minDispersionScoreServerSet.add(primaryRS); + this.minDispersionScore = dispersionScore; + } else if (dispersionScore == this.minDispersionScore) { + this.minDispersionScoreServerSet.add(primaryRS); + } + + // Update the min dispersion num + if (dispersionNum < this.minDispersionNum) { + this.minDispersionNumServerSet.clear(); + this.minDispersionNumServerSet.add(primaryRS); + this.minDispersionNum = dispersionNum; + } else if (dispersionNum == this.minDispersionNum) { + this.minDispersionNumServerSet.add(primaryRS); + } + + dispersionScoreSummary += dispersionScore; + dispersionNumSummary += dispersionNum; + } + + // Update the avg dispersion score + if (primaryRSToRegionCounterMap.keySet().size() != 0) { + this.avgDispersionScore = dispersionScoreSummary / + (float) primaryRSToRegionCounterMap.keySet().size(); + this.avgDispersionNum = dispersionNumSummary / + (float) primaryRSToRegionCounterMap.keySet().size(); + } + } + + /** + * @return list which contains just 3 elements: average dispersion score, max + * dispersion score and min dispersion score as first, second and third element + * respectively. + * + */ + public List getDispersionInformation() { + List dispersion = new ArrayList(); + dispersion.add(avgDispersionScore); + dispersion.add(maxDispersionScore); + dispersion.add(minDispersionScore); + return dispersion; + } + + public void print(boolean isDetailMode) { + if (!isFilledUp) { + System.err.println("[Error] Region assignment verfication report" + + "hasn't been filled up"); + } + DecimalFormat df = new java.text.DecimalFormat( "#.##"); + + // Print some basic information + System.out.println("Region Assignment Verification for Table: " + tableName + + "\n\tTotal regions : " + totalRegions); + + // Print the number of regions on each kinds of the favored nodes + System.out.println("\tTotal regions on favored nodes " + + totalFavoredAssignments); + for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) { + System.out.println("\t\tTotal regions on "+ p.toString() + + " region servers: " + favoredNodes[p.ordinal()]); + } + // Print the number of regions in each kinds of invalid assignment + System.out.println("\tTotal unassigned regions: " + + unAssignedRegionsList.size()); + if (isDetailMode) { + for (HRegionInfo region : unAssignedRegionsList) { + System.out.println("\t\t" + region.getRegionNameAsString()); + } + } + + System.out.println("\tTotal regions NOT on favored nodes: " + + nonFavoredAssignedRegionList.size()); + if (isDetailMode) { + for (HRegionInfo region : nonFavoredAssignedRegionList) { + System.out.println("\t\t" + region.getRegionNameAsString()); + } + } + + System.out.println("\tTotal regions without favored nodes: " + + regionsWithoutValidFavoredNodes.size()); + if (isDetailMode) { + for (HRegionInfo region : regionsWithoutValidFavoredNodes) { + System.out.println("\t\t" + region.getRegionNameAsString()); + } + } + + // Print the locality information if enabled + if (this.enforceLocality && totalRegions != 0) { + // Print the actual locality for this table + float actualLocality = 100 * + this.actualLocalitySummary / (float) totalRegions; + System.out.println("\n\tThe actual avg locality is " + + df.format(actualLocality) + " %"); + + // Print the expected locality if regions are placed on the each kinds of + // favored nodes + for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) { + float avgLocality = 100 * + (favoredNodesLocalitySummary[p.ordinal()] / (float) totalRegions); + System.out.println("\t\tThe expected avg locality if all regions" + + " on the " + p.toString() + " region servers: " + + df.format(avgLocality) + " %"); + } + } + + // Print the region balancing information + System.out.println("\n\tTotal hosting region servers: " + + totalRegionServers); + // Print the region balance information + if (totalRegionServers != 0) { + System.out.println( + "\tAvg dispersion num: " +df.format(avgDispersionNum) + + " hosts;\tMax dispersion num: " + df.format(maxDispersionNum) + + " hosts;\tMin dispersion num: " + df.format(minDispersionNum) + + " hosts;"); + + System.out.println("\t\tThe number of the region servers with the max" + + " dispersion num: " + this.maxDispersionNumServerSet.size()); + if (isDetailMode) { + printHServerAddressSet(maxDispersionNumServerSet); + } + + System.out.println("\t\tThe number of the region servers with the min" + + " dispersion num: " + this.minDispersionNumServerSet.size()); + if (isDetailMode) { + printHServerAddressSet(maxDispersionNumServerSet); + } + + System.out.println( + "\tAvg dispersion score: " + df.format(avgDispersionScore) + + ";\tMax dispersion score: " + df.format(maxDispersionScore) + + ";\tMin dispersion score: " + df.format(minDispersionScore) + ";"); + + System.out.println("\t\tThe number of the region servers with the max" + + " dispersion score: " + this.maxDispersionScoreServerSet.size()); + if (isDetailMode) { + printHServerAddressSet(maxDispersionScoreServerSet); + } + + System.out.println("\t\tThe number of the region servers with the min" + + " dispersion score: " + this.minDispersionScoreServerSet.size()); + if (isDetailMode) { + printHServerAddressSet(minDispersionScoreServerSet); + } + + System.out.println( + "\tAvg regions/region server: " + df.format(avgRegionsOnRS) + + ";\tMax regions/region server: " + maxRegionsOnRS + + ";\tMin regions/region server: " + minRegionsOnRS + ";"); + + // Print the details about the most loaded region servers + System.out.println("\t\tThe number of the most loaded region servers: " + + mostLoadedRSSet.size()); + if (isDetailMode) { + printHServerAddressSet(mostLoadedRSSet); + } + + // Print the details about the least loaded region servers + System.out.println("\t\tThe number of the least loaded region servers: " + + leastLoadedRSSet.size()); + if (isDetailMode) { + printHServerAddressSet(leastLoadedRSSet); + } + } + System.out.println("=============================="); + } + + /** + * Return the unassigned regions + * @return unassigned regions + */ + List getUnassignedRegions() { + return unAssignedRegionsList; + } + + /** + * Return the regions without favored nodes + * @return regions without favored nodes + */ + List getRegionsWithoutValidFavoredNodes() { + return regionsWithoutValidFavoredNodes; + } + + /** + * Return the regions not assigned to its favored nodes + * @return regions not assigned to its favored nodes + */ + List getNonFavoredAssignedRegions() { + return nonFavoredAssignedRegionList; + } + + /** + * Return the number of regions assigned to their favored nodes + * @return number of regions assigned to their favored nodes + */ + int getTotalFavoredAssignments() { + return totalFavoredAssignments; + } + + /** + * Return the number of regions based on the position (primary/secondary/ + * tertiary) assigned to their favored nodes + * @param position + * @return the number of regions + */ + int getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position position) { + return favoredNodes[position.ordinal()]; + } + + private void printHServerAddressSet(Set serverSet) { + if (serverSet == null) { + return ; + } + int i = 0; + for (ServerName addr : serverSet){ + if ((i++) % 3 == 0) { + System.out.print("\n\t\t\t"); + } + System.out.print(addr.getHostAndPort() + " ; "); + } + System.out.println("\n"); + } +} diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/FavoredGroupLoadBalancer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/FavoredGroupLoadBalancer.java new file mode 100644 index 0000000..dc0788c --- /dev/null +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/FavoredGroupLoadBalancer.java @@ -0,0 +1,369 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rsgroup; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Random; +import java.util.Set; + +import com.google.common.net.HostAndPort; +import com.google.common.collect.Sets; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.StartcodeAgnosticServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.favored.FavoredNodesManager; +import org.apache.hadoop.hbase.master.RackManager; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper; +import org.apache.hadoop.hbase.master.balancer.FavoredNodesPromoter; +import org.apache.hadoop.hbase.master.balancer.FavoredStochasticBalancer; +import org.apache.hadoop.util.StringUtils; + +import com.google.common.collect.Lists; + +public class FavoredGroupLoadBalancer extends RSGroupBasedLoadBalancer implements + FavoredNodesPromoter { + private static final Log LOG = LogFactory.getLog(FavoredGroupLoadBalancer.class); + + private FavoredNodesPromoter favoredNodesPromoter; + private RackManager rackManager; + + @Override + public void initialize() throws HBaseIOException { + + internalBalancer = new FavoredStochasticBalancer(); + internalBalancer.setConf(config); + internalBalancer.setMasterServices(masterServices); + internalBalancer.setClusterStatus(clusterStatus); + internalBalancer.initialize(); + favoredNodesPromoter = (FavoredNodesPromoter) internalBalancer; + if (RSGroupInfoManager == null) { + initializeRSGroupManager(); + } + rackManager = new RackManager(config); + } + + @Override + public Map> redistribute( + Map> clusterState) throws IOException { + if (!isOnline()) { + throw new IllegalStateException(RSGroupInfoManager.RSGROUP_TABLE_NAME + + " is not online, unable to perform balance"); + } + Map> result = new HashMap>(); + for (RSGroupInfo info : RSGroupInfoManager.listRSGroups()) { + Map> groupClusterState = new HashMap>(); + for (HostAndPort hostAndPort : info.getServers()) { + for (ServerName curr : clusterState.keySet()) { + if (ServerName.isSameHostnameAndPort(curr, ServerName.valueOf(hostAndPort, ServerName.NON_STARTCODE))) { + groupClusterState.put(curr, clusterState.get(curr)); + } + } + } + if (groupClusterState.size() > 0) { + Map> groupFavoredNodes = favoredNodesPromoter + .redistribute(groupClusterState); + if (groupFavoredNodes != null) { + result.putAll(groupFavoredNodes); + } + } + } + return result; + } + + // TODO - Check if implementation and plans are correct + @Override + public List completeRedistribute( + Map> clusterState) throws IOException { + if (!isOnline()) { + throw new IllegalStateException(RSGroupInfoManager.RSGROUP_TABLE_NAME + + " is not online, unable to perform balance"); + } + List result = Lists.newArrayList(); + try { + for (RSGroupInfo info : RSGroupInfoManager.listRSGroups()) { + Map> groupClusterState = new HashMap>(); + for (HostAndPort sName : info.getServers()) { + for (ServerName curr : clusterState.keySet()) { + if (ServerName.isSameHostnameAndPort(curr, ServerName.valueOf(sName, ServerName.NON_STARTCODE))) { + groupClusterState.put(curr, clusterState.get(curr)); + } + } + } + List regionsToRR = Lists.newArrayList(); + for (List hris : groupClusterState.values()) { + regionsToRR.addAll(hris); + } + if (groupClusterState.size() > 0) { + LOG.debug("Doing complete redistribute for group : " + info.getName()); + List partialResult = favoredNodesPromoter + .completeRedistribute(groupClusterState); + if (partialResult != null) { + result.addAll(partialResult); + } + } + } + } catch (IOException exp) { + LOG.warn("Exception while redistributing cluster.", exp); + result.clear(); + } + return result; + } + + @Override + public Map> checkFavoredNodes(List servers, + List regions) { + Map> result = new HashMap>(); + try { + for (RSGroupInfo info : RSGroupInfoManager.listRSGroups()) { + result.putAll(checkGroupFavoredNodes(info.getName(), servers)); + } + } catch (IOException exp) { + LOG.warn(StringUtils.stringifyException(exp)); + } + return result; + } + + public Map> checkGroupFavoredNodes(String groupName, + List servers) throws IOException { + RSGroupInfo info = RSGroupInfoManager.getRSGroup(groupName); + Map> result = new HashMap>(); + List groupServers = filterOfflineServers(info, servers); + List groupRegions = Lists.newArrayList(); + for (TableName tName : info.getTables()) { + groupRegions.addAll(this.masterServices.getAssignmentManager().getRegionStates() + .getRegionsOfTable(tName)); + } + if (groupServers.size() > 0) { + LOG.debug("checking favored nodes for group : " + info.getName()); + Map> partialResult = favoredNodesPromoter.checkFavoredNodes( + groupServers, groupRegions); + if (partialResult != null) { + result.putAll(partialResult); + } + } else { + for (HostAndPort hostAndPort : info.getServers()) { + List empty = Lists.newArrayList(); + result.put(ServerName.valueOf(hostAndPort, ServerName.NON_STARTCODE), empty); + } + LOG.warn("The entire region server group " + info.getName() + " is dead."); + } + return result; + } + + @Override + public Map> generateFavoredNodesForDaughter( + List servers, HRegionInfo parent, HRegionInfo hriA, HRegionInfo hriB) + throws IOException { + + Set regionAFavNodes = Sets.newHashSet(); + Set regionBFavNodes = Sets.newHashSet(); + Map> result = new HashMap>(); + TableName table = parent.getTable(); + + String group = RSGroupInfoManager.getRSGroupOfTable(table); + List groupServers = + getGroupServerList(RSGroupInfoManager.getRSGroup(group)); + + // Use all servers in group instead of onlineservers. That will help avoid failures when only one + // or two servers are online in a group. + FavoredNodeAssignmentHelper helper = + new FavoredNodeAssignmentHelper(groupServers, rackManager); + helper.initialize(); + + FavoredNodesManager fnm = masterServices.getFavoredNodesManager(); + List parentFavoredNodes = fnm.getFavoredNodes(parent); + if (parentFavoredNodes == null) { + LOG.debug("Unable to find favored nodes for parent, " + parent + + " generating new favored nodes for daughter"); + regionAFavNodes = Sets.newHashSet(helper.generateFavoredNodes(hriA)); + regionBFavNodes = Sets.newHashSet(helper.generateFavoredNodes(hriB)); + + } else { + + LOG.debug("FN of parent " + parent.getRegionNameAsString() + " " + parentFavoredNodes); + List groupFavoredNodes = getServersInGroup(groupServers, parentFavoredNodes); + List nonGroupFavoredNodes = getServersNotInGroup(groupServers, parentFavoredNodes); + + // Let both daughters have the same primary from parent nodes in the same group whether table is moved/not + ServerName primary = getRNServerWithExclusion(parentFavoredNodes, Sets.newHashSet(nonGroupFavoredNodes)); + if (primary != null) { + regionAFavNodes.add(primary); + regionBFavNodes.add(primary); + } + + if (nonGroupFavoredNodes.size() == 0) { + + addRNFavNodeWithExclusion(regionAFavNodes, parentFavoredNodes, regionAFavNodes); + addRNFavNodeWithExclusion(regionBFavNodes, parentFavoredNodes, regionAFavNodes); + + } else { + LOG.debug("Nodes:" + nonGroupFavoredNodes + " does not belong to server list, Table:" + + parent.getTable().getNameAsString() + " likely being moved"); + LOG.debug("Nodes:" + groupFavoredNodes + " belong to the same group for table: " + + parent.getTable().getNameAsString()); + + addRNFavNodeWithExclusion(regionAFavNodes, nonGroupFavoredNodes, regionAFavNodes); + addRNFavNodeWithExclusion(regionBFavNodes, nonGroupFavoredNodes, regionBFavNodes); + } + + assignMissingFavoredNodes(regionAFavNodes, helper, parentFavoredNodes); + assignMissingFavoredNodes(regionBFavNodes, helper, parentFavoredNodes); + } + + LOG.debug("FN of daughter-1- " + hriA.getRegionNameAsString() + " " + regionAFavNodes); + LOG.debug("FN of daughter-2- " + hriB.getRegionNameAsString() + " " + regionBFavNodes); + + result.put(hriA, Lists.newArrayList(regionAFavNodes)); + result.put(hriB, Lists.newArrayList(regionBFavNodes)); + fnm.updateFavoredNodes(hriA, result.get(hriA)); + fnm.updateFavoredNodes(hriB, result.get(hriB)); + return result; + } + + private void assignMissingFavoredNodes(Set regionFN, + FavoredNodeAssignmentHelper helper, List parentFavoredNodes) throws IOException { + + try { + regionFN.add(helper.generateMissingFavoredNode(Lists.newArrayList(regionFN), parentFavoredNodes)); + } catch (IOException e) { + LOG.warn("Could not generate FN for " + regionFN + " trying without excludenodes:" + parentFavoredNodes); + regionFN.add(helper.generateMissingFavoredNode(Lists.newArrayList(regionFN))); + } + } + + private void addRNFavNodeWithExclusion(Set regionNodes, List selectionNodes, + Set excludeNodes) throws IOException { + ServerName sn = getRNServerWithExclusion(selectionNodes, excludeNodes); + if (sn != null) { + regionNodes.add(sn); + } + } + + private ServerName getRNServerWithExclusion(List parentFavoredNodes, + Set excludeNodes) throws IOException { + + List nodesToConsider = Lists.newArrayList(parentFavoredNodes); + nodesToConsider.removeAll(excludeNodes); + if (nodesToConsider.size() > 0) { + return getRandomServerFromList(nodesToConsider); + } + return null; + } + + private ServerName getRandomServerFromList(List nodesToConsider) { + Random random = new Random(); + return nodesToConsider.get(random.nextInt(nodesToConsider.size())); + } + + private List getGroupServerList(RSGroupInfo group) { + List groupServerList = Lists.newArrayList(); + for (HostAndPort hostAndPort : group.getServers()) { + groupServerList.add(ServerName.valueOf(hostAndPort, ServerName.NON_STARTCODE)); + } + return groupServerList; + } + + private List getServerNamesWithoutStartCode(List groupServers) { + List groupServerList = Lists.newArrayList(); + for (ServerName sn : groupServers) { + groupServerList.add(ServerName.valueOf(sn.getHostAndPort(), ServerName.NON_STARTCODE)); + } + return groupServerList; + } + + private List getServersInGroup(List groupServers, + List parentFavoredNodes) { + + List serversInGroup = Lists.newArrayList(parentFavoredNodes); + List groupServerList = getServerNamesWithoutStartCode(groupServers); + serversInGroup.retainAll(groupServerList); + return serversInGroup; + } + + private List getServersNotInGroup(List groupServers, + List parentFavoredNodes) { + + List serversNotInList = Lists.newArrayList(parentFavoredNodes); + List groupServerList = getServerNamesWithoutStartCode(groupServers); + serversNotInList.removeAll(groupServerList); + return serversNotInList; + } + + @Override + public void generateFavoredNodesForMergedRegion(HRegionInfo merged, HRegionInfo hriA, + HRegionInfo hriB) throws IOException { + favoredNodesPromoter.generateFavoredNodesForMergedRegion(merged, hriA, hriB); + } + + public void removeFavoredFromGroup(HostAndPort hostAndPort, String destinationGroup, + List onlineServers) throws IOException { + RSGroupInfo group = RSGroupInfoManager.getRSGroup(destinationGroup); + if (group != null) { + List newServers = filterServers(group.getServers(), onlineServers); + masterServices.getFavoredNodesManager().removeFavoredNode( + ServerName.valueOf(hostAndPort, ServerName.NON_STARTCODE), newServers); + } else { + throw new IOException(destinationGroup + " does not exist."); + } + } + + @Override + public void setClusterStatus(ClusterStatus st) { + super.setClusterStatus(st); + if (this.internalBalancer != null) { + this.internalBalancer.setClusterStatus(st); + } + } + + public boolean checkForFavoredNodesInTargetGroup(TableName table, String destinationGroup, + List onlineGroupServers) throws IOException { + boolean matched = true; + List regions = this.masterServices.getAssignmentManager().getRegionStates() + .getRegionsOfTable(table); + for (HRegionInfo hri : regions) { + if (hri.getTable().isSystemTable()) continue; + List fns = masterServices.getFavoredNodesManager().getFavoredNodes(hri); + boolean regionMatchedFN = false; + for (ServerName sn : fns) { + if (onlineGroupServers.contains(StartcodeAgnosticServerName.valueOf(sn))) { + regionMatchedFN = true; + break; + } + } + matched = matched && regionMatchedFN; + if (!matched) { + LOG.debug("Found region with no FNs in target group " + hri.getRegionNameAsString()); + return false; + } + } + return matched; + } +} diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java index f94d0f6..be9c84c 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdmin.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.List; import java.util.Set; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Connection; @@ -118,4 +119,30 @@ public abstract class RSGroupAdmin implements Closeable { * @throws java.io.IOException on unexpected failure to retrieve GroupInfo */ public abstract RSGroupInfo getRSGroupOfServer(HostAndPort hostPort) throws IOException; + + /** + * Redistributes the replicas of regions belonging to a group. + * This API does not change assignment of regions, but might + * do an update to the favored nodes in META. + * The conditions considering a replica for move are: + * 1. The replica is not the current assignment. + * 2. The other two replicas have data locality greater than + * specified by FavoredNodesPromoter.MIN_LOCALITY_FOR_REDISTRIBUTE + * @param name the name of the group to balance + * @return true, if successful + * @throws IOException Signals that an I/O exception has occurred. + */ + public abstract boolean redistribute(String name) throws IOException; + + /** + * Completely redistributes region replicas in a group. + * This API does a load balance of all the 3*region count + * replicas. + * @param name the name of the group to balance + * @return true, if successful + * @throws IOException Signals that an I/O exception has occurred. + */ + public abstract boolean completeRedistribute(String name) throws IOException; + + public abstract List checkGroupFavoredNodes(String groupName) throws IOException; } diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java index a7f14f7..ca7eab3 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminClient.java @@ -19,6 +19,7 @@ */ package org.apache.hadoop.hbase.rsgroup; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.net.HostAndPort; import com.google.protobuf.ServiceException; @@ -30,6 +31,7 @@ import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; @@ -39,6 +41,10 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos; import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfServerResponse; import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.GetRSGroupInfoOfTableResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RedistributeGroupRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesRequest; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CheckGroupFavoredNodesResponse; +import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.CompleteRedistributeGroupRequest; import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos; @@ -199,6 +205,46 @@ class RSGroupAdminClient extends RSGroupAdmin { } @Override + public boolean redistribute(String name) throws IOException { + RedistributeGroupRequest request = RedistributeGroupRequest + .newBuilder().setGroupName(name).build(); + try { + return proxy.redistributeGroup(null, request).getResult(); + } catch (ServiceException e) { + throw ProtobufUtil.getRemoteException(e); + } + } + + @Override + public boolean completeRedistribute(String name) throws IOException { + CompleteRedistributeGroupRequest request = CompleteRedistributeGroupRequest + .newBuilder().setGroupName(name).build(); + try { + return proxy.completeRedistributeGroup(null, request).getResult(); + } catch (ServiceException e) { + throw ProtobufUtil.getRemoteException(e); + } + } + + @Override + public List checkGroupFavoredNodes(String groupName) throws IOException { + CheckGroupFavoredNodesRequest request = CheckGroupFavoredNodesRequest + .newBuilder().setGroupName(groupName).build(); + List result = Lists.newArrayList(); + try { + CheckGroupFavoredNodesResponse response = proxy.checkGroupFavoredNodes( + null, request); + for (org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName sn : response + .getServersList()) { + result.add(ProtobufUtil.toServerName(sn)); + } + } catch (ServiceException e) { + throw ProtobufUtil.getRemoteException(e); + } + return result; + } + + @Override public void close() throws IOException { } } diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java index 22bad72..13cc1f2 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java @@ -78,7 +78,7 @@ import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveTablesR import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RSGroupAdminService; import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupRequest; import org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupResponse; - +import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos; public class RSGroupAdminEndpoint extends RSGroupAdminService implements CoprocessorService, Coprocessor, MasterObserver { @@ -259,6 +259,61 @@ public class RSGroupAdminEndpoint extends RSGroupAdminService done.run(response); } + @Override public void redistributeGroup(RpcController controller, + RSGroupAdminProtos.RedistributeGroupRequest request, + RpcCallback done) { + RSGroupAdminProtos.RedistributeGroupResponse response = null; + RSGroupAdminProtos.RedistributeGroupResponse.Builder builder = + RSGroupAdminProtos.RedistributeGroupResponse.newBuilder(); + boolean result = false; + try { + result = groupAdminServer.redistribute(request.getGroupName()); + } catch (IOException e) { + LOG.debug( + "Caught Exception while doing redistributeGroup for group " + request.getGroupName(), e); + ResponseConverter.setControllerException(controller, e); + } + builder.setResult(result); + response = builder.build(); + done.run(response); + } + + @Override public void completeRedistributeGroup(RpcController controller, + RSGroupAdminProtos.CompleteRedistributeGroupRequest request, + RpcCallback done) { + RSGroupAdminProtos.CompleteRedistributeGroupResponse response = null; + RSGroupAdminProtos.CompleteRedistributeGroupResponse.Builder builder = + RSGroupAdminProtos.CompleteRedistributeGroupResponse.newBuilder(); + boolean result = false; + try { + result = groupAdminServer.completeRedistribute(request.getGroupName()); + } catch (IOException e) { + LOG.debug( + "Caught Exception while doing completeRedistributeGroup for group " + + request.getGroupName(), e); + ResponseConverter.setControllerException(controller, e); + } + builder.setResult(result); + response = builder.build(); + done.run(response); + } + + @Override public void checkGroupFavoredNodes(RpcController controller, + RSGroupAdminProtos.CheckGroupFavoredNodesRequest request, + RpcCallback done) { + RSGroupAdminProtos.CheckGroupFavoredNodesResponse.Builder response = + RSGroupAdminProtos.CheckGroupFavoredNodesResponse.newBuilder(); + try { + List deadServers = groupAdminServer.checkGroupFavoredNodes(request.getGroupName()); + for (ServerName sn : deadServers) { + response.addServers(ProtobufUtil.toServerName(sn)); + } + } catch (IOException e) { + ResponseConverter.setControllerException(controller, e); + } + done.run(response.build()); + } + @Override public void getRSGroupInfoOfServer(RpcController controller, GetRSGroupInfoOfServerRequest request, diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java index 8725781..23c496d 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminServer.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.StartcodeAgnosticServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.constraint.ConstraintException; @@ -54,6 +55,9 @@ import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.ServerManager; import org.apache.hadoop.hbase.master.TableLockManager.TableLock; +import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper; +import org.apache.hadoop.hbase.master.balancer.FavoredNodesPromoter; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; /** * Service to support Region Server Grouping (HBase-6721) @@ -231,6 +235,14 @@ public class RSGroupAdminServer extends RSGroupAdmin { if (master.getMasterCoprocessorHost() != null) { master.getMasterCoprocessorHost().postMoveServers(servers, targetGroupName); } + if (this.master.getFavoredNodesManager() != null) { + FavoredGroupLoadBalancer loadBalancer = ((FavoredGroupLoadBalancer) this.master + .getLoadBalancer()); + for (HostAndPort host : servers) { + loadBalancer.removeFavoredFromGroup(host, srcGrp.getName(), master.getServerManager() + .getOnlineServersList()); + } + } LOG.info("Move server done: "+sourceGroupName+"->"+targetGroupName); } } @@ -246,6 +258,10 @@ public class RSGroupAdminServer extends RSGroupAdmin { return; } RSGroupInfoManager manager = getRSGroupInfoManager(); + FavoredGroupLoadBalancer fnm = null; + if (master.getLoadBalancer() instanceof FavoredGroupLoadBalancer) { + fnm = ((FavoredGroupLoadBalancer) this.master.getLoadBalancer()); + } synchronized (manager) { if (master.getMasterCoprocessorHost() != null) { master.getMasterCoprocessorHost().preMoveTables(tables, targetGroup); @@ -256,8 +272,20 @@ public class RSGroupAdminServer extends RSGroupAdmin { if(destGroup == null) { throw new ConstraintException("Target group does not exist: "+targetGroup); } - if(destGroup.getServers().size() < 1) { - throw new ConstraintException("Target group must have at least one server."); + if(destGroup.getServers().size() < FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) { + throw new ConstraintException("Target group must have at least " + + FavoredNodeAssignmentHelper.FAVORED_NODES_NUM + " servers."); + } + if (fnm != null) { + List groupServers = FavoredNodeAssignmentHelper + .filterServers(destGroup.getServers(), master.getServerManager().getOnlineServersList()); + + for(TableName table : tables) { + if (!fnm.checkForFavoredNodesInTargetGroup(table, targetGroup, groupServers)) { + throw new ConstraintException( + "Atleast one favored node has to be migrated to the new group for table :" + table); + } + } } } @@ -373,16 +401,20 @@ public class RSGroupAdminServer extends RSGroupAdmin { plans.addAll(partialPlans); } } - long startTime = System.currentTimeMillis(); + long startTime = EnvironmentEdgeManager.currentTime(); balancerRan = plans != null; if (plans != null && !plans.isEmpty()) { LOG.info("Group balance "+groupName+" starting with plan count: "+plans.size()); for (RegionPlan plan: plans) { - LOG.info("balance " + plan); - assignmentManager.balance(plan); + LOG.trace("balance " + plan); + if (LoadBalancer.BOGUS_SERVER_NAME.equals(plan.getDestination())) { + assignmentManager.unassign(plan.getRegionInfo()); + } else { + assignmentManager.balance(plan); + } } - LOG.info("Group balance "+groupName+" completed after "+ - (System.currentTimeMillis()-startTime)+" seconds"); + LOG.info("Group balance " + groupName + " completed after " + + (EnvironmentEdgeManager.currentTime() - startTime) + " seconds"); } if (master.getMasterCoprocessorHost() != null) { master.getMasterCoprocessorHost().postBalanceRSGroup(groupName, balancerRan); @@ -401,6 +433,110 @@ public class RSGroupAdminServer extends RSGroupAdmin { return getRSGroupInfoManager().getRSGroupOfServer(hostPort); } + @Override + public boolean redistribute(String name) throws IOException { + if (!master.isInitialized()) { + throw new IOException( + "Master has not been initialized, cannot run redistribute."); + } + ServerManager serverManager = master.getServerManager(); + FavoredNodesPromoter balancer = (FavoredNodesPromoter) master.getLoadBalancer(); + if (balancer == null) { + LOG.debug("Balancer not an instance of FavoredNodesPromoter."); + return false; + } + long startTime = System.currentTimeMillis(); + LOG.info("Starting to redistribute"); + synchronized (balancer) { + // Only allow one balance run at at time. + Map groupRIT = rsGroupGetRegionsInTransition(name); + if (groupRIT.size() > 0) { + LOG.debug("Not running redistribute because " + + groupRIT.size() + + " region(s) in transition : " + + StringUtils.abbreviate(master.getAssignmentManager().getRegionStates() + .getRegionsInTransition().toString(), 256)); + return false; + } + if (serverManager.areDeadServersInProgress()) { + LOG.debug("Not running redistribute because processing dead regionserver(s): " + + serverManager.getDeadServers()); + return false; + } + Map> assignment = getRSGroupAssignments(name); + Map> updatedFavoredNodes = balancer + .redistribute(assignment); + LOG.info(" Redistribute resulted in update of favored nodes for regions : " + + updatedFavoredNodes.size()); + for (HRegionInfo hri : updatedFavoredNodes.keySet()) { + master.getFavoredNodesManager().updateFavoredNodes(hri, updatedFavoredNodes.get(hri)); + } + master.getFavoredNodesManager().updateFavoredNodesInRegionServer(updatedFavoredNodes); + } + LOG.info("Group redistribute " + name + " completed after " + + (System.currentTimeMillis() - startTime) + " seconds"); + return true; + } + + @Override + public boolean completeRedistribute(String name) throws IOException { + if (!master.isInitialized()) { + throw new IOException( + "Master has not been initialized, cannot run completeRedistribute."); + } + ServerManager serverManager = master.getServerManager(); + FavoredNodesPromoter balancer = (FavoredNodesPromoter) master.getLoadBalancer(); + if (balancer == null) { + LOG.debug("Balancer not an instance of FavoredNodesPromoter."); + return false; + } + long startTime = EnvironmentEdgeManager.currentTime(); + LOG.info("Starting to complete redistribute"); + synchronized (balancer) { + // Only allow one balance run at at time. + Map groupRIT = rsGroupGetRegionsInTransition(name); + if (groupRIT.size() > 0) { + LOG.debug("Not running complete redistribute because " + + groupRIT.size() + + " region(s) in transition : " + + StringUtils.abbreviate(master.getAssignmentManager().getRegionStates() + .getRegionsInTransition().toString(), 256)); + return false; + } + if (serverManager.areDeadServersInProgress()) { + LOG.debug("Not running complete redistribute because processing dead regionserver(s): " + + serverManager.getDeadServers()); + return false; + } + Map> assignment = getRSGroupAssignments(name); + List regionPlans = balancer.completeRedistribute(assignment); + if (regionPlans != null && !regionPlans.isEmpty()) { + for (RegionPlan plan : regionPlans) { + master.getAssignmentManager().balance(plan); + } + } + LOG.info("Group complete redistribute " + name + " completed after " + + (EnvironmentEdgeManager.currentTime() - startTime) + " seconds"); + return true; + } + } + + @Override + public List checkGroupFavoredNodes(String groupName) throws IOException { + List result = Lists.newArrayList(); + if (this.master.getFavoredNodesManager() != null) { + Map> deadServers = ((FavoredGroupLoadBalancer) this.master + .getLoadBalancer()).checkGroupFavoredNodes(groupName, master.getServerManager() + .getOnlineServersList()); + for (Map.Entry> entry : deadServers.entrySet()) { + LOG.info(entry.getKey() + " is dead and referenced by the regions = " + + StringUtils.join(entry.getValue().iterator(), ",")); + result.add(entry.getKey()); + } + } + return result; + } + @InterfaceAudience.Private public RSGroupInfoManager getRSGroupInfoManager() throws IOException { return RSGroupInfoManager; @@ -423,6 +559,26 @@ public class RSGroupAdminServer extends RSGroupAdmin { return rit; } + private Map> getRSGroupAssignments(String groupName) + throws IOException { + Map> result = Maps.newHashMap(); + RSGroupInfo RSGroupInfo = getRSGroupInfo(groupName); + for (ServerName serverName : master.getServerManager().getOnlineServers().keySet()) { + if (RSGroupInfo.getServers().contains(serverName.getHostPort())) { + result.put(serverName, new ArrayList()); + } + } + for (Map.Entry entry : master.getAssignmentManager() + .getRegionStates().getRegionAssignments().entrySet()) { + ServerName currServer = entry.getValue(); + HRegionInfo currRegion = entry.getKey(); + if (result.containsKey(currServer)) { + result.get(currServer).add(currRegion); + } + } + return result; + } + private Map>> getRSGroupAssignmentsByTable(String groupName) throws IOException { Map>> result = Maps.newHashMap(); diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java index c4b4bf3..0c8c814 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java @@ -77,11 +77,11 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer, LoadBalanc private static final Log LOG = LogFactory.getLog(RSGroupBasedLoadBalancer.class); - private Configuration config; - private ClusterStatus clusterStatus; - private MasterServices masterServices; - private RSGroupInfoManager RSGroupInfoManager; - private LoadBalancer internalBalancer; + protected Configuration config; + protected ClusterStatus clusterStatus; + protected MasterServices masterServices; + protected RSGroupInfoManager RSGroupInfoManager; + protected LoadBalancer internalBalancer; //used during reflection by LoadBalancerFactory @InterfaceAudience.Private @@ -269,7 +269,7 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer, LoadBalanc } } - private List filterOfflineServers(RSGroupInfo RSGroupInfo, + protected List filterOfflineServers(RSGroupInfo RSGroupInfo, List onlineServers) { if (RSGroupInfo != null) { return filterServers(RSGroupInfo.getServers(), onlineServers); @@ -288,7 +288,7 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer, LoadBalanc * List of servers which are online. * @return the list */ - private List filterServers(Collection servers, + protected List filterServers(Collection servers, Collection onlineServers) { ArrayList finalList = new ArrayList(); for (HostAndPort server : servers) { @@ -370,14 +370,7 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer, LoadBalanc public void initialize() throws HBaseIOException { try { if (RSGroupInfoManager == null) { - List cps = - masterServices.getMasterCoprocessorHost().findCoprocessors(RSGroupAdminEndpoint.class); - if (cps.size() != 1) { - String msg = "Expected one implementation of GroupAdminEndpoint but found " + cps.size(); - LOG.error(msg); - throw new HBaseIOException(msg); - } - RSGroupInfoManager = cps.get(0).getGroupInfoManager(); + initializeRSGroupManager(); } } catch (IOException e) { throw new HBaseIOException("Failed to initialize GroupInfoManagerImpl", e); @@ -394,6 +387,17 @@ public class RSGroupBasedLoadBalancer implements RSGroupableBalancer, LoadBalanc internalBalancer.initialize(); } + protected void initializeRSGroupManager() throws HBaseIOException { + List cps = + masterServices.getMasterCoprocessorHost().findCoprocessors(RSGroupAdminEndpoint.class); + if (cps.size() != 1) { + String msg = "Expected one implementation of GroupAdminEndpoint but found " + cps.size(); + LOG.error(msg); + throw new HBaseIOException(msg); + } + RSGroupInfoManager = cps.get(0).getGroupInfoManager(); + } + public boolean isOnline() { return RSGroupInfoManager != null && RSGroupInfoManager.isOnline(); } diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RegionPlacementMaintainer.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RegionPlacementMaintainer.java new file mode 100644 index 0000000..062648a --- /dev/null +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RegionPlacementMaintainer.java @@ -0,0 +1,1703 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.rsgroup; + +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.NavigableMap; +import java.util.Random; +import java.util.Scanner; +import java.util.Set; +import java.util.TreeMap; + +import com.google.common.net.HostAndPort; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ClusterConnection; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.HConnection; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.constraint.ConstraintException; +import org.apache.hadoop.hbase.master.RackManager; +import org.apache.hadoop.hbase.master.SnapshotOfRegionAssignmentFromMeta; +import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper; +import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.RequestConverter; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService.BlockingInterface; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesRequest; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesResponse; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.FavoredNodesInfoPair; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ReplicaLoad; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerReplicaLoadPair; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetFavoredNodesForRegionRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetFavoredNodesForRegionResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetFavoredNodesForTableRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetFavoredNodesForTableResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetReplicaLoadRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.GetReplicaLoadResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MasterService; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.MunkresAssignment; +import org.apache.hadoop.hbase.util.Pair; + +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import com.google.protobuf.ServiceException; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +/** + * A tool that is used for manipulating and viewing favored nodes information + * for regions. Run with -h to get a list of the options + */ +@InterfaceAudience.Private +// TODO: Remove? Unused. Partially implemented only. +public class RegionPlacementMaintainer { + private static final Log LOG = LogFactory.getLog(RegionPlacementMaintainer.class + .getName()); + //The cost of a placement that should never be assigned. + private static final float MAX_COST = Float.POSITIVE_INFINITY; + + // The cost of a placement that is undesirable but acceptable. + private static final float AVOID_COST = 100000f; + + // The amount by which the cost of a placement is increased if it is the + // last slot of the server. This is done to more evenly distribute the slop + // amongst servers. + private static final float LAST_SLOT_COST_PENALTY = 0.5f; + + // The amount by which the cost of a primary placement is penalized if it is + // not the host currently serving the region. This is done to minimize moves. + private static final float NOT_CURRENT_HOST_PENALTY = 0.1f; + + private static boolean USE_MUNKRES_FOR_PLACING_SECONDARY_AND_TERTIARY = false; + + private Configuration conf; + private final boolean enforceLocality; + private final boolean enforceMinAssignmentMove; + private Admin admin; + private RSGroupAdmin groupAdmin; + private RackManager rackManager; + private Set targetTableSet; + private MasterService.BlockingInterface master; + private final Connection connection; + + public RegionPlacementMaintainer(Configuration conf) throws IOException { + this(conf, true, true); + } + + public RegionPlacementMaintainer(Configuration conf, boolean enforceLocality, + boolean enforceMinAssignmentMove) throws IOException { + this.conf = conf; + this.enforceLocality = enforceLocality; + this.enforceMinAssignmentMove = enforceMinAssignmentMove; + this.targetTableSet = new HashSet(); + this.rackManager = new RackManager(conf); + try { + this.connection = ConnectionFactory.createConnection(this.conf); + } catch (IOException e) { + throw new RuntimeException(e); + } + this.master = ((ClusterConnection)connection).getMaster(); + this.admin = connection.getAdmin(); + } + + private static void printHelp(Options opt) { + new HelpFormatter().printHelp( + "RegionPlacement < -w | -u | -n | -v | -t | -h | -overwrite -r regionName -f favoredNodes " + + "-diff>" + + " [-l false] [-m false] [-d] [-tables t1,t2,...tn] [-zk zk1,zk2,zk3]" + + " [-fs hdfs://a.b.c.d:9000] [-hbase_root /HBASE]", opt); + } + + public void setTargetTableName(String[] tableNames) { + if (tableNames != null) { + for (String table : tableNames) { + this.targetTableSet.add(TableName.valueOf(table)); + } + } + } + + private RSGroupAdmin getGroupAdmin() throws IOException { + if (this.groupAdmin == null) { + this.groupAdmin = new RSGroupAdminClient(admin.getConnection()); + } + return this.groupAdmin; + } + + private List getFavoredNodes(HRegionInfo hri) + throws ServiceException { + List favNodes = Lists.newArrayList(); + GetFavoredNodesForRegionRequest.Builder request = GetFavoredNodesForRegionRequest.newBuilder(); + request.setRegionInfo(HRegionInfo.convert(hri)); + GetFavoredNodesForRegionResponse response = + master.getFavoredNodesForRegion(null, request.build()); + for (org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName sn : response + .getServersList()) { + favNodes.add(ProtobufUtil.toServerName(sn)); + } + return favNodes; + } + + private List getServers(String groupName) throws IOException { + List result = new ArrayList(); + RSGroupInfo gInfo = getGroupAdmin().getRSGroupInfo(groupName); + if (gInfo == null) { + throw new IOException("Group information for group : " + groupName + " found null."); + } else { + for (HostAndPort sn : gInfo.getServers()) { + result.add(ServerName.valueOf(sn, ServerName.NON_STARTCODE)); + } + } + return result; + } + + private boolean isGroupMultiRack(TableName table) throws IOException { + Set racks = Sets.newHashSet(); + RSGroupInfo gInfo = getGroupAdmin().getRSGroupInfoOfTable(table); + if (gInfo == null) { + throw new IOException("Group information for table : " + table + " found null."); + } else { + for (HostAndPort sn : gInfo.getServers()) { + racks.add(this.rackManager.getRack(ServerName.valueOf(sn, ServerName.NON_STARTCODE))); + } + } + return racks.size() > 1; + } + + private RackManager getRackManager() { + return this.rackManager; + } + /** + * @return the new RegionAssignmentSnapshot + * @throws IOException + */ + public SnapshotOfRegionAssignmentFromMeta getRegionAssignmentSnapshot() + throws IOException { + SnapshotOfRegionAssignmentFromMeta currentAssignmentShapshot = + new SnapshotOfRegionAssignmentFromMeta(admin.getConnection()); + new SnapshotOfRegionAssignmentFromMeta(ConnectionFactory.createConnection(conf)); + currentAssignmentShapshot.initialize(); + return currentAssignmentShapshot; + } + + /** + * Verify the region placement is consistent with the assignment plan + * @param isDetailMode + * @return reports + * @throws IOException + */ + public List verifyRegionPlacement(boolean isDetailMode) + throws IOException { + System.out.println("Start to verify the region assignment and " + + "generate the verification report"); + // Get the region assignment snapshot + SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot(); + + // Get all the tables + Set tables = snapshot.getTableSet(); + + // Get the region locality map + Map> regionLocalityMap = null; + if (this.enforceLocality == true) { + regionLocalityMap = FSUtils.getRegionDegreeLocalityMappingFromFS(conf); + } + List reports = new ArrayList(); + // Iterate all the tables to fill up the verification report + for (TableName table : tables) { + if (!this.targetTableSet.isEmpty() && + !this.targetTableSet.contains(table)) { + continue; + } + AssignmentVerificationReport report = new AssignmentVerificationReport(); + report.fillUp(table, snapshot, regionLocalityMap); + report.print(isDetailMode); + reports.add(report); + } + return reports; + } + + /** + * Generate the assignment plan for the existing table + * + * @param tableName + * @param assignmentSnapshot + * @param regionLocalityMap + * @param plan + * @param munkresForSecondaryAndTertiary if set on true the assignment plan + * for the tertiary and secondary will be generated with Munkres algorithm, + * otherwise will be generated using placeSecondaryAndTertiaryRS + * @throws IOException + */ + private void genAssignmentPlan(TableName tableName, + SnapshotOfRegionAssignmentFromMeta assignmentSnapshot, + Map> regionLocalityMap, FavoredNodesPlan plan, + boolean munkresForSecondaryAndTertiary) throws IOException { + // Get the all the regions for the current table + List regions = + assignmentSnapshot.getTableToRegionMap().get(tableName); + int numRegions = regions.size(); + + // Get the current assignment map + Map currentAssignmentMap = + assignmentSnapshot.getRegionToRegionServerMap(); + + // Get the all the region servers + List servers = new ArrayList(); + try (Admin admin = this.connection.getAdmin()) { + servers.addAll(admin.getClusterStatus().getServers()); + } + servers.addAll(getServers(null)); + + LOG.info("Start to generate assignment plan for " + numRegions + + " regions from table " + tableName + " with " + + servers.size() + " region servers"); + + int slotsPerServer = (int) Math.ceil((float) numRegions / + servers.size()); + int regionSlots = slotsPerServer * servers.size(); + + // Compute the primary, secondary and tertiary costs for each region/server + // pair. These costs are based only on node locality and rack locality, and + // will be modified later. + float[][] primaryCost = new float[numRegions][regionSlots]; + float[][] secondaryCost = new float[numRegions][regionSlots]; + float[][] tertiaryCost = new float[numRegions][regionSlots]; + + if (this.enforceLocality && regionLocalityMap != null) { + // Transform the locality mapping into a 2D array, assuming that any + // unspecified locality value is 0. + float[][] localityPerServer = new float[numRegions][regionSlots]; + for (int i = 0; i < numRegions; i++) { + Map serverLocalityMap = + regionLocalityMap.get(regions.get(i).getEncodedName()); + if (serverLocalityMap == null) { + continue; + } + for (int j = 0; j < servers.size(); j++) { + String serverName = servers.get(j).getHostname(); + if (serverName == null) { + continue; + } + Float locality = serverLocalityMap.get(serverName); + if (locality == null) { + continue; + } + for (int k = 0; k < slotsPerServer; k++) { + // If we can't find the locality of a region to a server, which occurs + // because locality is only reported for servers which have some + // blocks of a region local, then the locality for that pair is 0. + localityPerServer[i][j * slotsPerServer + k] = locality.floatValue(); + } + } + } + + // Compute the total rack locality for each region in each rack. The total + // rack locality is the sum of the localities of a region on all servers in + // a rack. + Map> rackRegionLocality = + new HashMap>(); + for (int i = 0; i < numRegions; i++) { + HRegionInfo region = regions.get(i); + for (int j = 0; j < regionSlots; j += slotsPerServer) { + String rack = rackManager.getRack(servers.get(j / slotsPerServer)); + Map rackLocality = rackRegionLocality.get(rack); + if (rackLocality == null) { + rackLocality = new HashMap(); + rackRegionLocality.put(rack, rackLocality); + } + Float localityObj = rackLocality.get(region); + float locality = localityObj == null ? 0 : localityObj.floatValue(); + locality += localityPerServer[i][j]; + rackLocality.put(region, locality); + } + } + for (int i = 0; i < numRegions; i++) { + for (int j = 0; j < regionSlots; j++) { + String rack = rackManager.getRack(servers.get(j / slotsPerServer)); + Float totalRackLocalityObj = + rackRegionLocality.get(rack).get(regions.get(i)); + float totalRackLocality = totalRackLocalityObj == null ? + 0 : totalRackLocalityObj.floatValue(); + + // Primary cost aims to favor servers with high node locality and low + // rack locality, so that secondaries and tertiaries can be chosen for + // nodes with high rack locality. This might give primaries with + // slightly less locality at first compared to a cost which only + // considers the node locality, but should be better in the long run. + primaryCost[i][j] = 1 - (2 * localityPerServer[i][j] - + totalRackLocality); + + // Secondary cost aims to favor servers with high node locality and high + // rack locality since the tertiary will be chosen from the same rack as + // the secondary. This could be negative, but that is okay. + secondaryCost[i][j] = 2 - (localityPerServer[i][j] + totalRackLocality); + + // Tertiary cost is only concerned with the node locality. It will later + // be restricted to only hosts on the same rack as the secondary. + tertiaryCost[i][j] = 1 - localityPerServer[i][j]; + } + } + } + + if (this.enforceMinAssignmentMove && currentAssignmentMap != null) { + // We want to minimize the number of regions which move as the result of a + // new assignment. Therefore, slightly penalize any placement which is for + // a host that is not currently serving the region. + for (int i = 0; i < numRegions; i++) { + for (int j = 0; j < servers.size(); j++) { + ServerName currentAddress = currentAssignmentMap.get(regions.get(i)); + if (currentAddress != null && + !currentAddress.equals(servers.get(j))) { + for (int k = 0; k < slotsPerServer; k++) { + primaryCost[i][j * slotsPerServer + k] += NOT_CURRENT_HOST_PENALTY; + } + } + } + } + } + + // Artificially increase cost of last slot of each server to evenly + // distribute the slop, otherwise there will be a few servers with too few + // regions and many servers with the max number of regions. + for (int i = 0; i < numRegions; i++) { + for (int j = 0; j < regionSlots; j += slotsPerServer) { + primaryCost[i][j] += LAST_SLOT_COST_PENALTY; + secondaryCost[i][j] += LAST_SLOT_COST_PENALTY; + tertiaryCost[i][j] += LAST_SLOT_COST_PENALTY; + } + } + + RandomizedMatrix randomizedMatrix = new RandomizedMatrix(numRegions, + regionSlots); + primaryCost = randomizedMatrix.transform(primaryCost); + int[] primaryAssignment = new MunkresAssignment(primaryCost).solve(); + primaryAssignment = randomizedMatrix.invertIndices(primaryAssignment); + + // Modify the secondary and tertiary costs for each region/server pair to + // prevent a region from being assigned to the same rack for both primary + // and either one of secondary or tertiary. + for (int i = 0; i < numRegions; i++) { + int slot = primaryAssignment[i]; + String rack = rackManager.getRack(servers.get(slot / slotsPerServer)); + for (int k = 0; k < servers.size(); k++) { + if (!rackManager.getRack(servers.get(k)).equals(rack)) { + continue; + } + if (k == slot / slotsPerServer) { + // Same node, do not place secondary or tertiary here ever. + for (int m = 0; m < slotsPerServer; m++) { + secondaryCost[i][k * slotsPerServer + m] = MAX_COST; + tertiaryCost[i][k * slotsPerServer + m] = MAX_COST; + } + } else { + // Same rack, do not place secondary or tertiary here if possible. + for (int m = 0; m < slotsPerServer; m++) { + secondaryCost[i][k * slotsPerServer + m] = AVOID_COST; + tertiaryCost[i][k * slotsPerServer + m] = AVOID_COST; + } + } + } + } + if (munkresForSecondaryAndTertiary) { + randomizedMatrix = new RandomizedMatrix(numRegions, regionSlots); + secondaryCost = randomizedMatrix.transform(secondaryCost); + int[] secondaryAssignment = new MunkresAssignment(secondaryCost).solve(); + secondaryAssignment = randomizedMatrix.invertIndices(secondaryAssignment); + + // Modify the tertiary costs for each region/server pair to ensure that a + // region is assigned to a tertiary server on the same rack as its secondary + // server, but not the same server in that rack. + for (int i = 0; i < numRegions; i++) { + int slot = secondaryAssignment[i]; + String rack = rackManager.getRack(servers.get(slot / slotsPerServer)); + for (int k = 0; k < servers.size(); k++) { + if (k == slot / slotsPerServer) { + // Same node, do not place tertiary here ever. + for (int m = 0; m < slotsPerServer; m++) { + tertiaryCost[i][k * slotsPerServer + m] = MAX_COST; + } + } else { + if (rackManager.getRack(servers.get(k)).equals(rack)) { + continue; + } + // Different rack, do not place tertiary here if possible. + for (int m = 0; m < slotsPerServer; m++) { + tertiaryCost[i][k * slotsPerServer + m] = AVOID_COST; + } + } + } + } + + randomizedMatrix = new RandomizedMatrix(numRegions, regionSlots); + tertiaryCost = randomizedMatrix.transform(tertiaryCost); + int[] tertiaryAssignment = new MunkresAssignment(tertiaryCost).solve(); + tertiaryAssignment = randomizedMatrix.invertIndices(tertiaryAssignment); + + for (int i = 0; i < numRegions; i++) { + List favoredServers = + new ArrayList(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM); + ServerName s = servers.get(primaryAssignment[i] / slotsPerServer); + favoredServers.add(ServerName.valueOf(s.getHostname(), s.getPort(), + ServerName.NON_STARTCODE)); + + s = servers.get(secondaryAssignment[i] / slotsPerServer); + favoredServers.add(ServerName.valueOf(s.getHostname(), s.getPort(), + ServerName.NON_STARTCODE)); + + s = servers.get(tertiaryAssignment[i] / slotsPerServer); + favoredServers.add(ServerName.valueOf(s.getHostname(), s.getPort(), + ServerName.NON_STARTCODE)); + // Update the assignment plan + plan.updateAssignmentPlan(regions.get(i), favoredServers); + } + LOG.info("Generated the assignment plan for " + numRegions + + " regions from table " + tableName + " with " + + servers.size() + " region servers"); + LOG.info("Assignment plan for secondary and tertiary generated " + + "using MunkresAssignment"); + } else { + Map primaryRSMap = new HashMap(); + for (int i = 0; i < numRegions; i++) { + primaryRSMap.put(regions.get(i), servers.get(primaryAssignment[i] / slotsPerServer)); + } + FavoredNodeAssignmentHelper favoredNodeHelper = + new FavoredNodeAssignmentHelper(servers, conf); + favoredNodeHelper.initialize(); + Map secondaryAndTertiaryMap = + favoredNodeHelper.placeSecondaryAndTertiaryWithRestrictions(primaryRSMap); + for (int i = 0; i < numRegions; i++) { + List favoredServers = + new ArrayList(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM); + HRegionInfo currentRegion = regions.get(i); + ServerName s = primaryRSMap.get(currentRegion); + favoredServers.add(ServerName.valueOf(s.getHostname(), s.getPort(), + ServerName.NON_STARTCODE)); + + ServerName[] secondaryAndTertiary = + secondaryAndTertiaryMap.get(currentRegion); + s = secondaryAndTertiary[0]; + favoredServers.add(ServerName.valueOf(s.getHostname(), s.getPort(), + ServerName.NON_STARTCODE)); + + s = secondaryAndTertiary[1]; + favoredServers.add(ServerName.valueOf(s.getHostname(), s.getPort(), + ServerName.NON_STARTCODE)); + // Update the assignment plan + plan.updateAssignmentPlan(regions.get(i), favoredServers); + } + LOG.info("Generated the assignment plan for " + numRegions + + " regions from table " + tableName + " with " + + servers.size() + " region servers"); + LOG.info("Assignment plan for secondary and tertiary generated " + + "using placeSecondaryAndTertiaryWithRestrictions method"); + } + } + + public FavoredNodesPlan getNewAssignmentPlan() throws IOException { + // Get the current region assignment snapshot by scanning from the META + SnapshotOfRegionAssignmentFromMeta assignmentSnapshot = + this.getRegionAssignmentSnapshot(); + + // Get the region locality map + Map> regionLocalityMap = null; + if (this.enforceLocality) { + regionLocalityMap = FSUtils.getRegionDegreeLocalityMappingFromFS(conf); + } + // Initialize the assignment plan + FavoredNodesPlan plan = new FavoredNodesPlan(); + + // Get the table to region mapping + Map> tableToRegionMap = + assignmentSnapshot.getTableToRegionMap(); + LOG.info("Start to generate the new assignment plan for the " + + + tableToRegionMap.keySet().size() + " tables" ); + for (TableName table : tableToRegionMap.keySet()) { + try { + if (!this.targetTableSet.isEmpty() && + !this.targetTableSet.contains(table)) { + continue; + } + // TODO: maybe run the placement in parallel for each table + genAssignmentPlan(table, assignmentSnapshot, regionLocalityMap, plan, + USE_MUNKRES_FOR_PLACING_SECONDARY_AND_TERTIARY); + } catch (Exception e) { + LOG.error("Get some exceptions for placing primary region server" + + "for table " + table + " because " + e); + } + } + LOG.info("Finish to generate the new assignment plan for the " + + + tableToRegionMap.keySet().size() + " tables" ); + return plan; + } + + /** + * Some algorithms for solving the assignment problem may traverse workers or + * jobs in linear order which may result in skewing the assignments of the + * first jobs in the matrix toward the last workers in the matrix if the + * costs are uniform. To avoid this kind of clumping, we can randomize the + * rows and columns of the cost matrix in a reversible way, such that the + * solution to the assignment problem can be interpreted in terms of the + * original untransformed cost matrix. Rows and columns are transformed + * independently such that the elements contained in any row of the input + * matrix are the same as the elements in the corresponding output matrix, + * and each row has its elements transformed in the same way. Similarly for + * columns. + */ + protected static class RandomizedMatrix { + private final int rows; + private final int cols; + private final int[] rowTransform; + private final int[] rowInverse; + private final int[] colTransform; + private final int[] colInverse; + + /** + * Create a randomization scheme for a matrix of a given size. + * @param rows the number of rows in the matrix + * @param cols the number of columns in the matrix + */ + public RandomizedMatrix(int rows, int cols) { + this.rows = rows; + this.cols = cols; + Random random = new Random(); + rowTransform = new int[rows]; + rowInverse = new int[rows]; + for (int i = 0; i < rows; i++) { + rowTransform[i] = i; + } + // Shuffle the row indices. + for (int i = rows - 1; i >= 0; i--) { + int r = random.nextInt(i + 1); + int temp = rowTransform[r]; + rowTransform[r] = rowTransform[i]; + rowTransform[i] = temp; + } + // Generate the inverse row indices. + for (int i = 0; i < rows; i++) { + rowInverse[rowTransform[i]] = i; + } + + colTransform = new int[cols]; + colInverse = new int[cols]; + for (int i = 0; i < cols; i++) { + colTransform[i] = i; + } + // Shuffle the column indices. + for (int i = cols - 1; i >= 0; i--) { + int r = random.nextInt(i + 1); + int temp = colTransform[r]; + colTransform[r] = colTransform[i]; + colTransform[i] = temp; + } + // Generate the inverse column indices. + for (int i = 0; i < cols; i++) { + colInverse[colTransform[i]] = i; + } + } + + /** + * Copy a given matrix into a new matrix, transforming each row index and + * each column index according to the randomization scheme that was created + * at construction time. + * @param matrix the cost matrix to transform + * @return a new matrix with row and column indices transformed + */ + public float[][] transform(float[][] matrix) { + float[][] result = new float[rows][cols]; + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + result[rowTransform[i]][colTransform[j]] = matrix[i][j]; + } + } + return result; + } + + /** + * Copy a given matrix into a new matrix, transforming each row index and + * each column index according to the inverse of the randomization scheme + * that was created at construction time. + * @param matrix the cost matrix to be inverted + * @return a new matrix with row and column indices inverted + */ + public float[][] invert(float[][] matrix) { + float[][] result = new float[rows][cols]; + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + result[rowInverse[i]][colInverse[j]] = matrix[i][j]; + } + } + return result; + } + + /** + * Given an array where each element {@code indices[i]} represents the + * randomized column index corresponding to randomized row index {@code i}, + * create a new array with the corresponding inverted indices. + * @param indices an array of transformed indices to be inverted + * @return an array of inverted indices + */ + public int[] invertIndices(int[] indices) { + int[] result = new int[indices.length]; + for (int i = 0; i < indices.length; i++) { + result[rowInverse[i]] = colInverse[indices[i]]; + } + return result; + } + } + + /** + * Print the assignment plan to the system output stream + * @param plan + */ + public static void printAssignmentPlan(FavoredNodesPlan plan) { + if (plan == null) return; + LOG.info("========== Start to print the assignment plan ================"); + // sort the map based on region info + Map> assignmentMap = + new TreeMap>(plan.getAssignmentMap()); + + for (Map.Entry> entry : assignmentMap.entrySet()) { + + String serverList = FavoredNodeAssignmentHelper.getFavoredNodesAsString(entry.getValue()); + String regionName = entry.getKey().getRegionNameAsString(); + LOG.info("Region: " + regionName ); + LOG.info("Its favored nodes: " + serverList); + } + LOG.info("========== Finish to print the assignment plan ================"); + } + + /** + * Update the assignment plan into hbase:meta + * @param plan the assignments plan to be updated into hbase:meta + * @throws IOException if cannot update assignment plan in hbase:meta + */ + public void updateAssignmentPlanToMeta(Map> favoredNodesMap) + throws IOException { + try { + LOG.info("Start to update the master with the new favored nodes."); + List>> regionUpdateInfos = + new ArrayList>>(); + for (Entry> entry : favoredNodesMap.entrySet()) { + regionUpdateInfos.add( + new Pair>(entry.getKey(), entry.getValue())); + } + UpdateFavoredNodesRequest request = + RequestConverter.buildUpdateFavoredNodesRequest(regionUpdateInfos); + master.updateFavoredNodesForRegion(null, request); + } catch (Exception e) { + String message = "Failed to update master with the new assignment plan "; + LOG.error(message, e); + throw new IOException(message, e); + } + } + + /** + * Update the assignment plan to all the region servers + * @param plan + * @throws IOException + */ + private void updateAssignmentPlanToRegionServers(FavoredNodesPlan plan) + throws IOException{ + LOG.info("Start to update the region servers with the new assignment plan"); + // Get the region to region server map + Map> currentAssignment = + this.getRegionAssignmentSnapshot().getRegionServerToRegionMap(); + + // track of the failed and succeeded updates + int succeededNum = 0; + Map failedUpdateMap = + new HashMap(); + + for (Map.Entry> entry : + currentAssignment.entrySet()) { + List>> regionUpdateInfos = + new ArrayList>>(); + try { + // Keep track of the favored updates for the current region server + FavoredNodesPlan singleServerPlan = null; + // Find out all the updates for the current region server + for (HRegionInfo region : entry.getValue()) { + List favoredServerList = plan.getFavoredNodes(region); + if (favoredServerList != null && + favoredServerList.size() == FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) { + // Create the single server plan if necessary + if (singleServerPlan == null) { + singleServerPlan = new FavoredNodesPlan(); + } + // Update the single server update + singleServerPlan.updateAssignmentPlan(region, favoredServerList); + regionUpdateInfos.add( + new Pair>(region, favoredServerList)); + } + } + if (singleServerPlan != null) { + // Update the current region server with its updated favored nodes + BlockingInterface currentRegionServer = + ((ClusterConnection)this.connection).getAdmin(entry.getKey()); + UpdateFavoredNodesRequest request = + RequestConverter.buildUpdateFavoredNodesRequest(regionUpdateInfos); + + UpdateFavoredNodesResponse updateFavoredNodesResponse = + currentRegionServer.updateFavoredNodes(null, request); + LOG.info("Region server " + + ProtobufUtil.getServerInfo(null, currentRegionServer).getServerName() + + " has updated " + updateFavoredNodesResponse.getResponse() + " / " + + singleServerPlan.getAssignmentMap().size() + + " regions with the assignment plan"); + succeededNum ++; + } + } catch (Exception e) { + failedUpdateMap.put(entry.getKey(), e); + } + } + // log the succeeded updates + LOG.info("Updated " + succeededNum + " region servers with " + + "the new assignment plan"); + + // log the failed updates + int failedNum = failedUpdateMap.size(); + if (failedNum != 0) { + LOG.error("Failed to update the following + " + failedNum + + " region servers with its corresponding favored nodes"); + for (Map.Entry entry : + failedUpdateMap.entrySet() ) { + LOG.error("Failed to update " + entry.getKey().getHostAndPort() + + " because of " + entry.getValue().getMessage()); + } + } + } + + public void updateAssignmentPlan(FavoredNodesPlan plan) + throws IOException { + LOG.info("Start to update the new assignment plan for the hbase:meta table and" + + " the region servers"); + // Update the new assignment plan to META + updateAssignmentPlanToMeta(plan.getAssignmentMap()); + // Update the new assignment plan to Region Servers + updateAssignmentPlanToRegionServers(plan); + LOG.info("Finish to update the new assignment plan for the hbase:meta table and" + + " the region servers"); + } + + public void printFavoredNodeDistribution(String groupName) throws ServiceException { + try { + if (StringUtils.isEmpty(groupName)) { + LOG.info("Printing favored node information for all the servers in cluster"); + } else { + LOG.info("Printing favored node information for group : " + groupName); + } + List servers = getServers(groupName); + GetReplicaLoadRequest.Builder request = GetReplicaLoadRequest.newBuilder(); + for (ServerName sn : servers) { + request.addServers(ProtobufUtil.toServerName(sn)); + } + GetReplicaLoadResponse response = this.master.getReplicaLoad(null, request.build()); + for (ServerName sn : servers) { + ServerName serverWithNoStartCode = ServerName.valueOf(sn.getHostAndPort(), + ServerName.NON_STARTCODE); + ReplicaLoad load = getReplicaCount(response.getReplicaLoadList(), serverWithNoStartCode); + StringBuilder sb = new StringBuilder(sn.toString()); + sb.append(": primary load = " + load.getPrimaryReplicaCount()); + sb.append(", secondary load = " + load.getSecondaryReplicaCount()); + sb.append(", tertiary load = " + load.getTertiaryReplicaCount()); + LOG.info(sb.toString()); + } + } catch (IOException e) { + LOG.error(e); + } + } + + private ReplicaLoad getReplicaCount(List replicaLoadList, ServerName sn) { + for (ServerReplicaLoadPair pair : replicaLoadList) { + if (ServerName.isSameHostnameAndPort(sn, ProtobufUtil.toServerName(pair.getServer()))) { + return pair.getReplicaCount(); + } + } + return ReplicaLoad.newBuilder().setPrimaryReplicaCount(0).setSecondaryReplicaCount(0) + .setTertiaryReplicaCount(0).build(); + } + + public Multimap getTableLocality() throws IOException { + if (targetTableSet.size() == 0) { + throw new IOException("No tables specified."); + } + Table table = null; + LinkedListMultimap localityMap = LinkedListMultimap.create(); + for (TableName tName : targetTableSet) { + try { + table = connection.getTable(tName); + RegionLocator regionLocator = connection.getRegionLocator(tName); + for (HRegionLocation regionLocation : regionLocator.getAllRegionLocations()) { + List favNodes = getFavoredNodes(regionLocation.getRegionInfo()); + for (ServerName sn : favNodes) { + localityMap.put( + regionLocation.getRegionInfo(), + HRegion.computeHDFSBlocksDistribution(conf, table.getTableDescriptor(), + regionLocation.getRegionInfo()).getBlockLocalityIndex(sn.getHostname())); + } + } + } catch (ServiceException exp) { + throw new IOException(exp); + } finally { + if (table != null) { + table.close(); + } + } + } + return localityMap; + } + + public Map> getFavoredNodesOfTable(TableName tableName) + throws IOException, ServiceException { + Map> favNodesMap = new HashMap>(); + GetFavoredNodesForTableRequest.Builder request = GetFavoredNodesForTableRequest.newBuilder(); + request.setTableName(ProtobufUtil.toProtoTableName(tableName)); + GetFavoredNodesForTableResponse response = + this.master.getFavoredNodesForTable(null, request.build()); + for (FavoredNodesInfoPair favNodeInfo : response.getFavoredNodesInfoList()) { + List favNodes = Lists.newArrayList(); + for (org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName sn : favNodeInfo + .getServersList()) { + favNodes.add(ProtobufUtil.toServerName(sn)); + } + favNodesMap.put(favNodeInfo.getRegionName(), favNodes); + } + return favNodesMap; + } + + public void moveSecondaryAndTertiaryToNewGroup(String targetGroupName) throws IOException { + if (targetTableSet.size() == 0) { + throw new IOException("No tables specified."); + } + List servers = getServers(targetGroupName); + FavoredNodeAssignmentHelper helper = new FavoredNodeAssignmentHelper(servers, getRackManager()); + helper.initialize(); + if (!helper.canPlaceFavoredNodes()) { + throw new ConstraintException(" Target group " + targetGroupName + + " does not have enough nodes to generate favored nodes"); + } + Table table = null; +// HConnection connection = this.getHBaseAdmin().getConnection(); + for (TableName tName : targetTableSet) { + try { + LOG.info("Moving secondary and tertiary favored nodes of table : " + tName); + Map> newFavNodesMap = + new HashMap>(); + Map> currentAssignmentByServer = new HashMap>(); + table = connection.getTable(tName); + RegionLocator regionLocator = connection.getRegionLocator(tName); + for (HRegionLocation regionLocation : regionLocator.getAllRegionLocations()) { + List newFavNodes = + new ArrayList(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM); + ServerName current = regionLocation.getServerName(); + HRegionInfo hri = regionLocation.getRegionInfo(); + if (currentAssignmentByServer.containsKey(current)) { + currentAssignmentByServer.get(current).add(hri); + } else { + currentAssignmentByServer.put(current, Lists.newArrayList(hri)); + } + newFavNodes.add(current); + newFavNodes.add(helper.generateMissingFavoredNode(newFavNodes)); + newFavNodes.add(helper.generateMissingFavoredNode(newFavNodes)); + newFavNodesMap.put(hri, newFavNodes); + } + updateAssignmentPlanToMeta(newFavNodesMap); + // Now update the region servers with the new Favored nodes. + updateRSWithFNInfo(connection, newFavNodesMap, currentAssignmentByServer); + } catch (IOException exp) { + throw exp; + } catch (ServiceException e) { + throw new IOException(e); + } finally { + if (table != null) { + table.close(); + } + } + } + } + + public void moveTertiaryToNewGroup(String targetGroupName) throws IOException { + if (targetTableSet.size() == 0) { + throw new IOException("No tables specified."); + } + List servers = getServers(targetGroupName); + FavoredNodeAssignmentHelper helper = new FavoredNodeAssignmentHelper(servers, getRackManager()); + helper.initialize(); + if (!helper.canPlaceFavoredNodes()) { + throw new ConstraintException(" Target group " + targetGroupName + + " does not have enough nodes to generate favored nodes"); + } + Connection connection = this.admin.getConnection(); + HTable table = null; + Random random = new Random(); + for (TableName tName : targetTableSet) { + try { + LOG.info("Moving tertiary favored node of table : " + tName); + boolean isMultiRack = isGroupMultiRack(tName); + Map> favNodes = getFavoredNodesOfTable(tName); + Map> newFavNodesMap = new HashMap>(); + Map> currentAssignmentByServer = new HashMap>(); + RegionLocator regionLocator = connection.getRegionLocator(tName); + for (HRegionLocation regionLocation : regionLocator.getAllRegionLocations()) { + HRegionInfo hri = regionLocation.getRegionInfo(); + ServerName current = regionLocation.getServerName(); + if (currentAssignmentByServer.containsKey(current)) { + currentAssignmentByServer.get(current).add(hri); + } else { + currentAssignmentByServer.put(current, Lists.newArrayList(hri)); + } + List newFavNodes = new ArrayList( + FavoredNodeAssignmentHelper.FAVORED_NODES_NUM); + newFavNodes.add(current); + Iterator itr = favNodes.get(hri.getRegionNameAsString()).iterator(); + if (!isMultiRack) { + while (itr.hasNext()) { + String hostAndPort = itr.next().getHostAndPort(); + if (!hostAndPort.equals(current.getHostAndPort())) { + newFavNodes.add(ServerName.valueOf(hostAndPort, ServerName.NON_STARTCODE)); + break; + } + } + newFavNodes.add(helper.generateMissingFavoredNode(newFavNodes)); + } else { + String currentRack = getRackManager().getRack(current); + List offRackHosts = Lists.newArrayList(); + while (itr.hasNext()) { + ServerName fn = itr.next(); + String rack = getRackManager().getRack(fn); + if (!rack.equals(currentRack)) { + offRackHosts.add(fn.getHostAndPort()); + } + } + if (offRackHosts.size() == 1) { + newFavNodes.add(ServerName.valueOf(offRackHosts.get(0), ServerName.NON_STARTCODE)); + } else { + newFavNodes.add(ServerName.valueOf(offRackHosts.get(random.nextInt(offRackHosts.size())), + ServerName.NON_STARTCODE)); + } + newFavNodes.add(helper.generateMissingFavoredNode(newFavNodes)); + } + newFavNodesMap.put(hri, newFavNodes); + } + updateAssignmentPlanToMeta(newFavNodesMap); + // Now update the region servers with the new Favored nodes. + updateRSWithFNInfo(connection, newFavNodesMap, currentAssignmentByServer); + } catch (IOException exp) { + throw exp; + } catch (ServiceException e) { + throw new IOException(e); + } finally { + if (table != null) { + table.close(); + } + } + } + } + + private void updateRSWithFNInfo(Connection connection, + Map> newFavNodesMap, + Map> currentAssignmentByServer) + throws IOException, ServiceException { + for (Entry> entry : currentAssignmentByServer.entrySet()) { + BlockingInterface currentRegionServer = ((ClusterConnection)connection).getAdmin(entry.getKey()); + List>> regionUpdateInfos = + new ArrayList>>(); + for (HRegionInfo hri : entry.getValue()) { + List newFN = newFavNodesMap.get(hri); + regionUpdateInfos.add(new Pair>(hri, newFN)); + } + UpdateFavoredNodesRequest request = + RequestConverter.buildUpdateFavoredNodesRequest(regionUpdateInfos); + UpdateFavoredNodesResponse updateFavoredNodesResponse = + currentRegionServer.updateFavoredNodes(null, request); + LOG.info("Region server " + entry.getKey().getServerName() + + " has been updated " + updateFavoredNodesResponse.getResponse() + " / " + + regionUpdateInfos.size() + " regions with the new favored nodes."); + } + } + + /** + * Return how many regions will move per table since their primary RS will + * change + * + * @param newPlan - new AssignmentPlan + * @return how many primaries will move per table + */ + public Map getRegionsMovement(FavoredNodesPlan newPlan) + throws IOException { + Map movesPerTable = new HashMap(); + SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot(); + Map> tableToRegions = snapshot + .getTableToRegionMap(); + FavoredNodesPlan oldPlan = snapshot.getExistingAssignmentPlan(); + Set tables = snapshot.getTableSet(); + for (TableName table : tables) { + int movedPrimaries = 0; + if (!this.targetTableSet.isEmpty() + && !this.targetTableSet.contains(table)) { + continue; + } + List regions = tableToRegions.get(table); + for (HRegionInfo region : regions) { + List oldServers = oldPlan.getFavoredNodes(region); + List newServers = newPlan.getFavoredNodes(region); + if (oldServers != null && newServers != null) { + ServerName oldPrimary = oldServers.get(0); + ServerName newPrimary = newServers.get(0); + if (oldPrimary.compareTo(newPrimary) != 0) { + movedPrimaries++; + } + } + } + movesPerTable.put(table, movedPrimaries); + } + return movesPerTable; + } + + /** + * Compares two plans and check whether the locality dropped or increased + * (prints the information as a string) also prints the baseline locality + * + * @param movesPerTable - how many primary regions will move per table + * @param regionLocalityMap - locality map from FS + * @param newPlan - new assignment plan + * @throws IOException + */ + public void checkDifferencesWithOldPlan(Map movesPerTable, + Map> regionLocalityMap, FavoredNodesPlan newPlan) + throws IOException { + // localities for primary, secondary and tertiary + SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot(); + FavoredNodesPlan oldPlan = snapshot.getExistingAssignmentPlan(); + Set tables = snapshot.getTableSet(); + Map> tableToRegionsMap = snapshot.getTableToRegionMap(); + for (TableName table : tables) { + float[] deltaLocality = new float[3]; + float[] locality = new float[3]; + if (!this.targetTableSet.isEmpty() + && !this.targetTableSet.contains(table)) { + continue; + } + List regions = tableToRegionsMap.get(table); + System.out.println("=================================================="); + System.out.println("Assignment Plan Projection Report For Table: " + table); + System.out.println("\t Total regions: " + regions.size()); + System.out.println("\t" + movesPerTable.get(table) + + " primaries will move due to their primary has changed"); + for (HRegionInfo currentRegion : regions) { + Map regionLocality = regionLocalityMap.get(currentRegion + .getEncodedName()); + if (regionLocality == null) { + continue; + } + List oldServers = oldPlan.getFavoredNodes(currentRegion); + List newServers = newPlan.getFavoredNodes(currentRegion); + if (newServers != null && oldServers != null) { + int i=0; + for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) { + ServerName newServer = newServers.get(p.ordinal()); + ServerName oldServer = oldServers.get(p.ordinal()); + Float oldLocality = 0f; + if (oldServers != null) { + oldLocality = regionLocality.get(oldServer.getHostname()); + if (oldLocality == null) { + oldLocality = 0f; + } + locality[i] += oldLocality; + } + Float newLocality = regionLocality.get(newServer.getHostname()); + if (newLocality == null) { + newLocality = 0f; + } + deltaLocality[i] += newLocality - oldLocality; + i++; + } + } + } + DecimalFormat df = new java.text.DecimalFormat( "#.##"); + for (int i = 0; i < deltaLocality.length; i++) { + System.out.print("\t\t Baseline locality for "); + if (i == 0) { + System.out.print("primary "); + } else if (i == 1) { + System.out.print("secondary "); + } else if (i == 2) { + System.out.print("tertiary "); + } + System.out.println(df.format(100 * locality[i] / regions.size()) + "%"); + System.out.print("\t\t Locality will change with the new plan: "); + System.out.println(df.format(100 * deltaLocality[i] / regions.size()) + + "%"); + } + System.out.println("\t Baseline dispersion"); + printDispersionScores(table, snapshot, regions.size(), null, true); + System.out.println("\t Projected dispersion"); + printDispersionScores(table, snapshot, regions.size(), newPlan, true); + } + } + + public void printDispersionScores(TableName table, + SnapshotOfRegionAssignmentFromMeta snapshot, int numRegions, FavoredNodesPlan newPlan, + boolean simplePrint) { + if (!this.targetTableSet.isEmpty() && !this.targetTableSet.contains(table)) { + return; + } + AssignmentVerificationReport report = new AssignmentVerificationReport(); + report.fillUpDispersion(table, snapshot, newPlan); + List dispersion = report.getDispersionInformation(); + if (simplePrint) { + DecimalFormat df = new java.text.DecimalFormat("#.##"); + System.out.println("\tAvg dispersion score: " + + df.format(dispersion.get(0)) + " hosts;\tMax dispersion score: " + + df.format(dispersion.get(1)) + " hosts;\tMin dispersion score: " + + df.format(dispersion.get(2)) + " hosts;"); + } else { + LOG.info("For Table: " + table + " ; #Total Regions: " + numRegions + + " ; The average dispersion score is " + dispersion.get(0)); + } + } + + public void printLocalityAndDispersionForCurrentPlan( + Map> regionLocalityMap) throws IOException { + SnapshotOfRegionAssignmentFromMeta snapshot = this.getRegionAssignmentSnapshot(); + FavoredNodesPlan assignmentPlan = snapshot.getExistingAssignmentPlan(); + Set tables = snapshot.getTableSet(); + Map> tableToRegionsMap = snapshot + .getTableToRegionMap(); + for (TableName table : tables) { + float[] locality = new float[3]; + if (!this.targetTableSet.isEmpty() + && !this.targetTableSet.contains(table)) { + continue; + } + List regions = tableToRegionsMap.get(table); + for (HRegionInfo currentRegion : regions) { + Map regionLocality = regionLocalityMap.get(currentRegion + .getEncodedName()); + if (regionLocality == null) { + continue; + } + List servers = assignmentPlan.getFavoredNodes(currentRegion); + if (servers != null) { + int i = 0; + for (FavoredNodesPlan.Position p : FavoredNodesPlan.Position.values()) { + ServerName server = servers.get(p.ordinal()); + Float currentLocality = 0f; + if (servers != null) { + currentLocality = regionLocality.get(server.getHostname()); + if (currentLocality == null) { + currentLocality = 0f; + } + locality[i] += currentLocality; + } + i++; + } + } + } + for (int i = 0; i < locality.length; i++) { + String copy = null; + if (i == 0) { + copy = "primary"; + } else if (i == 1) { + copy = "secondary"; + } else if (i == 2) { + copy = "tertiary" ; + } + float avgLocality = 100 * locality[i] / regions.size(); + LOG.info("For Table: " + table + " ; #Total Regions: " + regions.size() + + " ; The average locality for " + copy+ " is " + avgLocality + " %"); + } + printDispersionScores(table, snapshot, regions.size(), null, false); + } + } + + /** + * Makes expansion on the new rack by migrating a number of regions from each of the existing + * racks to a new rack. Assumption: FavoredNodeAssignmentHelper and RackManager will contain + * information that the newly added rack is there (but no regions are assigned to it). + */ + public void expandRegionsToNewRack(String newRack, FavoredNodeAssignmentHelper domain) + throws IOException { + SnapshotOfRegionAssignmentFromMeta assignmentSnapshot = getRegionAssignmentSnapshot(); + FavoredNodesPlan plan = assignmentSnapshot.getExistingAssignmentPlan(); + /* + * List servers = new ArrayList(); + * servers.addAll(getHBaseAdmin().getClusterStatus().getServers()); FavoredNodeAssignmentHelper + * domain = new FavoredNodeAssignmentHelper(servers, rackManager); + */ + Map> mapServerToRegions = assignmentSnapshot + .getRegionServerToRegionMap(); + Map> mapRackToRegionServers = domain.getRackToRegionServerMap(); + + int avgRegionsPerRs = calculateAvgRegionsPerRS(mapServerToRegions); + if (avgRegionsPerRs == 0) { + System.out.println("ERROR: average number of regions per regionserver is ZERO, ABORTING!"); + return; + } + + // because the domain knows about the newly added rack we subtract 1. + int totalNumRacks = domain.getTotalNumberOfRacks() - 1; + int numServersNewRack = domain.getServersFromRack(newRack).size(); + int moveRegionsPerRack = (int) Math.floor(avgRegionsPerRs * numServersNewRack / totalNumRacks); + List regionsToMove = new ArrayList(); + for (String rack : mapRackToRegionServers.keySet()) { + if (!rack.equals(newRack)) { + List regionsToMoveFromRack = pickRegionsFromRack(plan, + mapRackToRegionServers.get(rack), mapServerToRegions, rack, moveRegionsPerRack, newRack); + if (regionsToMoveFromRack.isEmpty()) { + System.out.println("WARNING: number of regions to be moved from rack " + rack + + " is zero!"); + } + regionsToMove.addAll(regionsToMoveFromRack); + } + } + moveRegionsToNewRack(plan, domain, regionsToMove, newRack); + } + + /** + * Returns the average number of regions per regionserver + * + * @param mapServerToRegions + * @return + */ + private int calculateAvgRegionsPerRS( + Map> mapServerToRegions) { + double totalRegions = 0; + for (Entry> entry : mapServerToRegions + .entrySet()) { + totalRegions += entry.getValue().size(); + } + System.out.println("total regions: " + totalRegions); + return (int) Math.floor(totalRegions / mapServerToRegions.size()); + } + + /** + * Move the regions to the new rack, such that each server will get equal number of regions + * @param plan - the current assignment plan + * @param domain - the assignment domain + * @param regionsToMove - the regions that would be moved to the new rack regionserver per rack + * are picked to be moved in the new rack + * @param newRack - the new rack + * @throws IOException + */ + private void moveRegionsToNewRack(FavoredNodesPlan plan, FavoredNodeAssignmentHelper domain, + List regionsToMove, String newRack) throws IOException { + System.out.println("------------------------------------------"); + System.out + .println("Printing how many regions are planned to be assigned per" + + "region server in the new rack (" + newRack + ")"); + List serversFromNewRack = domain.getServersFromRack(newRack); + int totalNumRSNewRack = serversFromNewRack.size(); + for (int j = 0; j < totalNumRSNewRack; j++) { + int regionsPerRs = 0; + for (int i = j; i < regionsToMove.size(); i += totalNumRSNewRack) { + HRegionInfo region = regionsToMove.get(i); + List favoredNodes = plan.getFavoredNodes(region); + if (FavoredNodesPlan.replaceFavoredNodesServerWithNew(favoredNodes, + FavoredNodesPlan.Position.TERTIARY, serversFromNewRack.get(j))) { + plan.updateAssignmentPlan(region, favoredNodes); + + } + regionsPerRs++; + } + System.out.println("RS: " + serversFromNewRack.get(j).getHostname() + " got " + regionsPerRs + + "tertiary regions"); + } + System.out.println("Do you want to update the assignment plan with this changes (y/n): "); + Scanner s = new Scanner(System.in); + String input = s.nextLine().trim(); + s.close(); + if (input.toLowerCase().equals("y")) { + System.out.println("Updating assignment plan..."); + updateAssignmentPlanToMeta(plan.getAssignmentMap()); + updateAssignmentPlanToRegionServers(plan); + } else { + System.out.println("exiting without updating the assignment plan"); + } + } + + /** + * This method will pick regions from a given rack, such that these regions + * are going to be moved to the new rack later. The logic behind is: we move + * the regions' tertiaries into a new rack + * + * @param plan - the current assignment plan + * @param regionServersFromOneRack - region servers that belong to the given + * rack + * @param mapServerToRegions map contains the mapping from a region server to + * it's regions + * @param currentRack + * @param moveRegionsPerRack - how many regions we want to move per rack + * @param newRack - the new rack where we will move the tertiaries + * @return a complete list of regions that are going to be moved in the new + * rack + */ + private List pickRegionsFromRack(FavoredNodesPlan plan, + List regionServersFromOneRack, + Map> mapServerToRegions, + String currentRack, int moveRegionsPerRack, String newRack) { + // we want to move equal number of tertiaries per server + List regionsToMove = new ArrayList(); + System.out.println("------------------------------------------"); + System.out.println("Printing how many regions are planned to be moved per regionserver in rack " + currentRack); + // use Math.ceil to ensure that we move at least one region from some of the RS + // if we should move a number of regions smaller than the total number of + // servers in the rack + int movePerServer = (int) Math.ceil(moveRegionsPerRack + / regionServersFromOneRack.size()); + int totalMovedPerRack = 0; + int serverIndex = 0; + for (serverIndex = 0; serverIndex < regionServersFromOneRack.size(); serverIndex++) { + ServerName server = regionServersFromOneRack.get(serverIndex); + int totalMovedPerRs = 0; + // get all the regions on this server + List regions = mapServerToRegions.get(server); + for (HRegionInfo region : regions) { + List favoredNodes = plan.getFavoredNodes(region); + if (favoredNodes.size() != FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) { + System.out.println("WARNING!!! Number of favored nodes for region " + + region.getEncodedName() + " is not " + + FavoredNodeAssignmentHelper.FAVORED_NODES_NUM); + } + regionsToMove.add(region); + totalMovedPerRs++; + totalMovedPerRack++; + if (totalMovedPerRs == movePerServer + || totalMovedPerRack == moveRegionsPerRack) + break; + } + System.out.println(totalMovedPerRs + " tertiary regions from RS " + + regionServersFromOneRack.get(serverIndex).getHostname() + + " will be moved to a new rack"); + if (totalMovedPerRack == moveRegionsPerRack) + break; + } + while (serverIndex < regionServersFromOneRack.size()) { + System.out.println("0 tertiary regions from RS " + + regionServersFromOneRack.get(serverIndex).getHostname() + + " will be moved to a new rack"); + serverIndex++; + } + System.out.println("Total number of regions: " + totalMovedPerRack + + " in rack " + currentRack + " will move its tertiary to a new rack: " + + newRack); + System.out.println("------------------------------------------"); + return regionsToMove; + } + + /** + * @param favoredNodesStr The String of favored nodes + * @return the list of ServerName for the byte array of favored nodes. + */ + public static List getFavoredNodeList(String favoredNodesStr) { + String[] favoredNodesArray = StringUtils.split(favoredNodesStr, ","); + if (favoredNodesArray == null) + return null; + + List serverList = new ArrayList(); + for (String hostNameAndPort : favoredNodesArray) { + serverList.add(ServerName.valueOf(hostNameAndPort, ServerName.NON_STARTCODE)); + } + return serverList; + } + + public static void main(String args[]) throws Exception { + Options opt = new Options(); + opt.addOption("w", "write", false, "write the assignments to hbase:meta only"); + opt.addOption("u", "update", false, + "update the assignments to hbase:meta and RegionServers together"); + opt.addOption("n", "dry-run", false, "do not write assignments to META"); + opt.addOption("v", "verify", false, "verify current assignments against META"); + opt.addOption("p", "print", false, "print the current assignment plan in META"); + opt.addOption("h", "help", false, "print usage"); + opt.addOption("d", "verification-details", false, + "print the details of verification report"); + + opt.addOption("zk", true, "to set the zookeeper quorum"); + opt.addOption("fs", true, "to set HDFS"); + opt.addOption("hbase_root", true, "to set hbase_root directory"); + + opt.addOption("overwrite", false, + "overwrite the favored nodes for a single region," + + "for example: -update -r regionName -f server1:port,server2:port,server3:port"); + opt.addOption("r", true, "The region name that needs to be updated"); + opt.addOption("f", true, "The new favored nodes"); + + opt.addOption("tables", true, + "The list of table names splitted by ',' ;" + + "For example: -tables: t1,t2,...,tn"); + opt.addOption("l", "locality", true, "enforce the maxium locality"); + opt.addOption("m", "min-move", true, "enforce minium assignment move"); + opt.addOption("diff", false, "calculate difference between assignment plans"); + opt.addOption("munkres", false, + "use munkres to place secondaries and tertiaries"); + opt.addOption("ld", "locality-dispersion", false, "print locality and dispersion " + + "information for current plan"); + opt.addOption("dist", "distribution", false, + "prints the primary, secondary and teritiary region distrbution for all region servers."); + opt.addOption("exprack", "expand-with-rack", false, "expand the regions to a new rack"); + opt.addOption("usegroup", "use-region-server-group", true, "use region servers of specified group"); + opt.addOption("floc", "print-fav-node-locality", false, + "print locality on favored nodes for tables."); + opt.addOption( + "migrateTable", + "migrate-table-to-new-group", + false, + "migrate secondary and tertiary hosts of tables specified by -tables option to " + + "new group specified by -usegroup option"); + opt.addOption( + "migrateTertiary", + "migrate-only-tertiary-host-table-to-new-group", + false, + "migrate only tertiary host of tables specified by -tables option to " + + "new group specified by -usegroup option"); + try { + // Set the log4j + Logger.getLogger("org.apache.zookeeper").setLevel(Level.ERROR); + Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.ERROR); + Logger.getLogger("org.apache.hadoop.hbase.rsgroup.RegionPlacementMaintainer") + .setLevel(Level.INFO); + + CommandLine cmd = new GnuParser().parse(opt, args); + Configuration conf = HBaseConfiguration.create(); + + boolean enforceMinAssignmentMove = true; + boolean enforceLocality = true; + boolean verificationDetails = false; + String groupName = null; + + // Read all the options + if ((cmd.hasOption("l") && + cmd.getOptionValue("l").equalsIgnoreCase("false")) || + (cmd.hasOption("locality") && + cmd.getOptionValue("locality").equalsIgnoreCase("false"))) { + enforceLocality = false; + } + + if ((cmd.hasOption("m") && + cmd.getOptionValue("m").equalsIgnoreCase("false")) || + (cmd.hasOption("min-move") && + cmd.getOptionValue("min-move").equalsIgnoreCase("false"))) { + enforceMinAssignmentMove = false; + } + + if (cmd.hasOption("zk")) { + conf.set(HConstants.ZOOKEEPER_QUORUM, cmd.getOptionValue("zk")); + LOG.info("Setting the zk quorum: " + conf.get(HConstants.ZOOKEEPER_QUORUM)); + } + + if (cmd.hasOption("fs")) { + conf.set(FileSystem.FS_DEFAULT_NAME_KEY, cmd.getOptionValue("fs")); + LOG.info("Setting the HDFS: " + conf.get(FileSystem.FS_DEFAULT_NAME_KEY)); + } + + if (cmd.hasOption("hbase_root")) { + conf.set(HConstants.HBASE_DIR, cmd.getOptionValue("hbase_root")); + LOG.info("Setting the hbase root directory: " + conf.get(HConstants.HBASE_DIR)); + } + + // Create the region placement obj + RegionPlacementMaintainer rp = new RegionPlacementMaintainer(conf, enforceLocality, + enforceMinAssignmentMove); + + if (cmd.hasOption("d") || cmd.hasOption("verification-details")) { + verificationDetails = true; + } + + if (cmd.hasOption("tables")) { + String tableNameListStr = cmd.getOptionValue("tables"); + String[] tableNames = StringUtils.split(tableNameListStr, ","); + rp.setTargetTableName(tableNames); + } + + if (cmd.hasOption("munkres")) { + USE_MUNKRES_FOR_PLACING_SECONDARY_AND_TERTIARY = true; + } + + if (cmd.hasOption("usegroup")) { + groupName = cmd.getOptionValue("usegroup"); + } + + // Read all the modes + if (cmd.hasOption("v") || cmd.hasOption("verify")) { + // Verify the region placement. + rp.verifyRegionPlacement(verificationDetails); + } else if (cmd.hasOption("n") || cmd.hasOption("dry-run")) { + // Generate the assignment plan only without updating the hbase:meta and RS + FavoredNodesPlan plan = rp.getNewAssignmentPlan(); + printAssignmentPlan(plan); + } else if (cmd.hasOption("w") || cmd.hasOption("write")) { + // Generate the new assignment plan + FavoredNodesPlan plan = rp.getNewAssignmentPlan(); + // Print the new assignment plan + printAssignmentPlan(plan); + // Write the new assignment plan to META + rp.updateAssignmentPlanToMeta(plan.getAssignmentMap()); + } else if (cmd.hasOption("u") || cmd.hasOption("update")) { + // Generate the new assignment plan + FavoredNodesPlan plan = rp.getNewAssignmentPlan(); + // Print the new assignment plan + printAssignmentPlan(plan); + // Update the assignment to hbase:meta and Region Servers + rp.updateAssignmentPlan(plan); + } else if (cmd.hasOption("diff")) { + FavoredNodesPlan newPlan = rp.getNewAssignmentPlan(); + Map> locality = FSUtils + .getRegionDegreeLocalityMappingFromFS(conf); + Map movesPerTable = rp.getRegionsMovement(newPlan); + rp.checkDifferencesWithOldPlan(movesPerTable, locality, newPlan); + System.out.println("Do you want to update the assignment plan? [y/n]"); + Scanner s = new Scanner(System.in); + String input = s.nextLine().trim(); + if (input.equals("y")) { + System.out.println("Updating assignment plan..."); + rp.updateAssignmentPlan(newPlan); + } + s.close(); + } else if (cmd.hasOption("ld")) { + Map> locality = FSUtils + .getRegionDegreeLocalityMappingFromFS(conf); + rp.printLocalityAndDispersionForCurrentPlan(locality); + } else if (cmd.hasOption("p") || cmd.hasOption("print")) { + FavoredNodesPlan plan = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan(); + if (rp.targetTableSet.size() == 0) { + printAssignmentPlan(plan); + } else { + for (TableName tName : rp.targetTableSet) { + Map> favNodesMap = rp.getFavoredNodesOfTable(tName); + for (Entry> entry : favNodesMap.entrySet()) { + String serverList = + FavoredNodeAssignmentHelper.getFavoredNodesAsString(entry.getValue()); + LOG.info("Region: " + entry.getKey()); + LOG.info("Its favored nodes: " + serverList); + } + } + } + } else if (cmd.hasOption("overwrite")) { + if (!cmd.hasOption("f") || !cmd.hasOption("r")) { + throw new IllegalArgumentException("Please specify: " + + " -update -r regionName -f server1:port,server2:port,server3:port"); + } + + String regionName = cmd.getOptionValue("r"); + String favoredNodesStr = cmd.getOptionValue("f"); + LOG.info("Going to update the region " + regionName + " with the new favored nodes " + + favoredNodesStr); + List favoredNodes = null; + HRegionInfo regionInfo = + rp.getRegionAssignmentSnapshot().getRegionNameToRegionInfoMap().get(regionName); + if (regionInfo == null) { + LOG.error("Cannot find the region " + regionName + " from the META"); + } else { + try { + favoredNodes = getFavoredNodeList(favoredNodesStr); + } catch (IllegalArgumentException e) { + LOG.error("Cannot parse the invalid favored nodes because " + e); + } + FavoredNodesPlan newPlan = new FavoredNodesPlan(); + newPlan.updateAssignmentPlan(regionInfo, favoredNodes); + rp.updateAssignmentPlan(newPlan); + } + } else if (cmd.hasOption("dist")) { + rp.printFavoredNodeDistribution(groupName); + } else if (cmd.hasOption("exprack")) { + /* List servers = new ArrayList(); + servers.addAll(rp.getHBaseAdmin().getClusterStatus().getServers());*/ + FavoredNodeAssignmentHelper domain = new FavoredNodeAssignmentHelper(rp.getServers(groupName), + rp.getRackManager()); + System.out.println("List of current racks: "); + Set allRacks = domain.getRackToRegionServerMap().keySet(); + for (String rack : allRacks) { + System.out.println("\t" + + rack + + "\t number of Region Servers: " + + domain.getServersFromRack(rack) + .size()); + } + System.out + .println("Insert the name of the new rack (to which the migration should be done): "); + Scanner s = new Scanner(System.in); + String newRack = s.nextLine().trim(); + s.close(); + rp.expandRegionsToNewRack(newRack, domain); + } else if (cmd.hasOption("floc")) { + Multimap localities = rp.getTableLocality(); + for (HRegionInfo hri : localities.keySet()) { + Collection loc = localities.get(hri); + if (loc.size() == 3) { + Iterator itr = loc.iterator(); + LOG.info("Localities for region : " + hri.getEncodedName() + " on primary = " + + itr.next() + " on secondary = " + itr.next() + " on tertiary = " + itr.next()); + + } else { + LOG.warn("3 Favored nodes not found for region :" + hri.getEncodedName()); + } + } + } else if (cmd.hasOption("migrateTable")) { + if (StringUtils.isEmpty(groupName)) { + throw new IllegalArgumentException("Please specify group name to migrate."); + } + rp.moveSecondaryAndTertiaryToNewGroup(groupName); + } else if (cmd.hasOption("migrateTertiary")) { + if (StringUtils.isEmpty(groupName)) { + throw new IllegalArgumentException("Please specify group name to migrate."); + } + rp.moveTertiaryToNewGroup(groupName); + } else { + printHelp(opt); + } + } catch (ParseException e) { + printHelp(opt); + } + } +} diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestFNTableImport.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestFNTableImport.java new file mode 100644 index 0000000..6f3fe91 --- /dev/null +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestFNTableImport.java @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rsgroup; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.favored.FavoredNodesManager; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.HBaseFsck; +import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil; +import com.google.common.collect.Sets; +import org.junit.After; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/* + * This case tests a scenario when a cluster with tables is moved from GroupBasedLoadBalancer + * to FavoredGroupLoadBalancer one and the auto-FN info that is populated. + */ +@Category(SmallTests.class) +public class TestFNTableImport { + + private static final Log LOG = LogFactory.getLog(TestFNTableImport.class); + + private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static final Configuration conf = UTIL.getConfiguration(); + private static final int SLAVES = 3; + private static final int REGION_NUM = 20; + + @After + public void stopCluster() throws Exception { + UTIL.cleanupTestDir(); + UTIL.shutdownMiniCluster(); + } + + @Test + public void testTableCreation() throws Exception { + + Configuration conf = UTIL.getConfiguration(); + // Lets setup basic info for GroupBased balancer + conf.set( + HConstants.HBASE_MASTER_LOADBALANCER_CLASS, RSGroupBasedLoadBalancer.class.getName()); + conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, RSGroupAdminEndpoint.class.getName()); + + LOG.info("Starting up cluster"); + UTIL.startMiniCluster(SLAVES); + while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) { + Threads.sleep(1); + } + HBaseAdmin admin = UTIL.getHBaseAdmin(); + admin.setBalancerRunning(false, true); + + String tableName = "testFNImport"; + HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, Bytes.toBytes("a"), Bytes.toBytes("z"), REGION_NUM); + UTIL.waitTableAvailable(TableName.valueOf(tableName)); + + LOG.info("Shutting down cluster"); + UTIL.shutdownMiniHBaseCluster(); + + Thread.sleep(2000); + LOG.info("Starting cluster again with FN Balancer"); + UTIL.getConfiguration().set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + FavoredGroupLoadBalancer.class.getName()); + UTIL.restartHBaseCluster(SLAVES); + while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) { + Threads.sleep(1); + } + admin = UTIL.getHBaseAdmin(); + admin.setBalancerRunning(false, true); + + UTIL.waitTableAvailable(TableName.valueOf(tableName)); + + FavoredNodesManager fnm = UTIL.getHBaseCluster().getMaster().getFavoredNodesManager(); + + List regionsOfTable = admin.getTableRegions(TableName.valueOf(tableName)); + for (HRegionInfo rInfo : regionsOfTable) { + Set favNodes = Sets.newHashSet(fnm.getFavoredNodes(rInfo)); + assertNotNull(favNodes); + assertTrue("3 favored nodes not found.", favNodes.size() == 3); + for (ServerName fn : favNodes) { + assertEquals("StartCode invalid for:" + fn, ServerName.NON_STARTCODE, fn.getStartcode()); + } + } + verifyHBCKErrors(0); + } + + void verifyHBCKErrors(int errors) throws Exception { + // TODO: Fix fsck +// HBaseFsck hbck = HbckTestingUtil.checkFavoredNodes(conf); +// assertEquals(errors, hbck.getErrors().getErrorList().size()); + } +} diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestFavoredGroupBalancerRSFailover.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestFavoredGroupBalancerRSFailover.java new file mode 100644 index 0000000..0d96738 --- /dev/null +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestFavoredGroupBalancerRSFailover.java @@ -0,0 +1,243 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rsgroup; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; + +import com.google.common.net.HostAndPort; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.master.balancer.FavoredNodesRepairChore; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.StartcodeAgnosticServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.rsgroup.RSGroupBasedLoadBalancer; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.net.ScriptBasedMapping; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Lists; + +@Category(LargeTests.class) +public class TestFavoredGroupBalancerRSFailover { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String GROUP = "test"; + private static final String NAMESPACE = "nsp1"; + private static final int SLAVES = 6; + private static HBaseAdmin admin; + private int REGION_NUM = 5; + protected static Configuration conf; + private RSGroupAdmin groupAdmin; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + setUpOnce(); + conf.setBoolean("hbase.assignment.usezk", true); + } + + static void setUpOnce() throws Exception { + conf = TEST_UTIL.getConfiguration(); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + FavoredGroupLoadBalancer.class, LoadBalancer.class); + conf.setFloat("hbase.min.locality.redistribute", 0.0f); + conf.setBoolean("hbase.redistribute.even.on.same.rack", true); + conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1200000); + conf.set(DFSConfigKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, + ScriptBasedMapping.class.getName()); + conf.set("hbase.coprocessor.master.classes", + RSGroupAdminEndpoint.class.getName()); + conf.setInt("hbase.assignment.maximum.attempts", 3); + conf.set(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, ""+ SLAVES); + //Don't let chore run. + conf.setInt(FavoredNodesRepairChore.FAVORED_NODE_REPAIR_CHORE_FREQ, Integer.MAX_VALUE); + } + + @Before + public void startCluster() throws Exception { + TEST_UTIL.startMiniCluster(SLAVES); + TEST_UTIL.getDFSCluster().waitClusterUp(); + //wait for balancer to come online + TEST_UTIL.waitFor(60000*5 , new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return TEST_UTIL.getMiniHBaseCluster().getMaster().isInitialized() + && ((RSGroupBasedLoadBalancer) TEST_UTIL.getMiniHBaseCluster().getMaster() + .getLoadBalancer()).isOnline(); + } + }); + Connection connection = ConnectionFactory.createConnection(conf); + admin = TEST_UTIL.getHBaseAdmin(); + groupAdmin = new RSGroupAdminClient(connection); + addGroup(groupAdmin, GROUP, 3); + NamespaceDescriptor nsp = NamespaceDescriptor.create(NAMESPACE) + .addConfiguration(RSGroupInfo.NAMESPACEDESC_PROP_GROUP, GROUP).build(); + admin.createNamespace(nsp); + } + + @After + public void stopCluster() throws Exception { + TableName[] tables = admin.listTableNamesByNamespace(NAMESPACE); + for (TableName tName : tables) { + admin.disableTable(tName); + admin.deleteTable(tName); + } + admin.deleteNamespace(NAMESPACE); + removeGroup(groupAdmin, GROUP); + TEST_UTIL.cleanupTestDir(); + TEST_UTIL.shutdownMiniCluster(); + } + + private RSGroupInfo addGroup(RSGroupAdmin groupAdmin2, String groupName, int serverCount) + throws IOException, InterruptedException { + RSGroupInfo defaultInfo = groupAdmin2.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP); + assertTrue(defaultInfo != null); + assertTrue(defaultInfo.getServers().size() >= serverCount); + groupAdmin2.addRSGroup(groupName); + + Set set = new HashSet(); + for (HostAndPort server : defaultInfo.getServers()) { + if (set.size() == serverCount) { + break; + } + set.add(server); + } + groupAdmin2.moveServers(set, groupName); + RSGroupInfo result = groupAdmin2.getRSGroupInfo(groupName); + assertTrue(result.getServers().size() >= serverCount); + return result; + } + + private void removeGroup(RSGroupAdmin admin, String groupName) throws IOException { + RSGroupInfo groupInfo = admin.getRSGroupInfo(groupName); + admin.moveTables(groupInfo.getTables(), RSGroupInfo.DEFAULT_GROUP); + admin.moveServers(groupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP); + admin.removeRSGroup(groupName); + } + + @Test + public void testKillAllFavoredNodes() throws Exception { + String tableName = NAMESPACE + TableName.NAMESPACE_DELIM + "testOneRSDead"; + HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM); + RegionPlacementMaintainer rpm = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration()); + Map> favoredNodes = rpm.getFavoredNodesOfTable(TableName + .valueOf(tableName)); + List regions = admin.getTableRegions(TableName.valueOf(tableName)); + Random rand = new Random(); + final HRegionInfo candidateRegion = regions.get(rand.nextInt(regions.size())); + List regionFavNodes = Lists.newArrayList(); + for (ServerName sn : favoredNodes.get(candidateRegion.getRegionNameAsString())) { + regionFavNodes.add(StartcodeAgnosticServerName.valueOf(sn)); + } + assertEquals(3, regionFavNodes.size()); + final ServerName currentRS1 = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionServerOfRegion(candidateRegion); + killRS(getServerToKill(currentRS1), (SLAVES - 1)); + TEST_UTIL.waitFor(60000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + ServerName sn = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionServerOfRegion(candidateRegion); + return (sn != null && !ServerName.isSameHostnameAndPort(sn, currentRS1)); + } + }); + final ServerName currentRS2 = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionServerOfRegion(candidateRegion); + assertTrue("Region is not on one of the favored nodes.", + regionFavNodes.contains(StartcodeAgnosticServerName.valueOf(currentRS2))); + killRS(getServerToKill(currentRS2), (SLAVES - 2)); + TEST_UTIL.waitFor(60000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + ServerName sn = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionServerOfRegion(candidateRegion); + return (sn != null && !ServerName.isSameHostnameAndPort(sn, currentRS2)); + } + }); + final ServerName currentRS3 = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionServerOfRegion(candidateRegion); + assertTrue("Region is not on one of the favored nodes.", + regionFavNodes.contains(StartcodeAgnosticServerName.valueOf(currentRS3))); + killRS(getServerToKill(currentRS3), (SLAVES - 3)); + + TEST_UTIL.waitFor(60000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return (TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates() + .isRegionInTransition(candidateRegion)); + } + }); + + RegionState state = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionTransitionState(candidateRegion.getEncodedName()); + assertEquals(state.getState(), RegionState.State.FAILED_OPEN); + } + + ServerName getServerToKill(ServerName dead) throws IOException { + for (RegionServerThread rs : TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads()){ + ServerName sn = rs.getRegionServer().getServerName(); + if (sn.getHostAndPort().equals(dead.getHostAndPort())) { + return sn; + } + } + return null; + } + + private void killRS(ServerName sn, final int expectedOnlineServers) throws Exception { + TEST_UTIL.getHBaseCluster().killRegionServer(sn); + TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(sn, -1); + TEST_UTIL.waitFor(60000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return (TEST_UTIL.getHBaseAdmin().getClusterStatus().getServers().size() == expectedOnlineServers); + } + }); + } + +} diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestFavoredGroupLoadBalancer.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestFavoredGroupLoadBalancer.java new file mode 100644 index 0000000..99f9cd4 --- /dev/null +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestFavoredGroupLoadBalancer.java @@ -0,0 +1,911 @@ +/** + * Copyright The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rsgroup; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import com.google.common.net.HostAndPort; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.client.HRegionLocator; +import org.apache.hadoop.hbase.client.HTableInterface; +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.master.RegionStates; +import org.apache.hadoop.hbase.master.balancer.BalancerTestBase; +import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper; +import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan; +import org.apache.hadoop.hbase.master.balancer.FavoredNodesRepairChore; +import org.apache.hadoop.hbase.master.balancer.ServerAndLoad; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.StartcodeAgnosticServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter; +import org.apache.hadoop.hbase.Waiter.Predicate; +import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.favored.FavoredNodesManager; +//import org.apache.hadoop.hbase.rsgroup.RSGroupMasterObserver; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.net.ScriptBasedMapping; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +@Category(LargeTests.class) +public class TestFavoredGroupLoadBalancer extends BalancerTestBase { + private static final Log LOG = LogFactory.getLog(TestFavoredGroupLoadBalancer.class); + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String GROUP = "test"; + private static final String NAMESPACE = "nsp1"; + private static final int SLAVES = 8; + private static HBaseAdmin admin; + private int REGION_NUM = 5; + protected static Configuration conf; + private RSGroupAdmin groupAdmin; + private static boolean postSplit = false; + + @BeforeClass + public static void setupBeforeClass() throws Exception { + setUpOnce(); + } + + static void setUpOnce() throws Exception { + conf = TEST_UTIL.getConfiguration(); + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + FavoredGroupLoadBalancer.class, LoadBalancer.class); + conf.setFloat("hbase.min.locality.redistribute", 0.0f); + conf.setBoolean("hbase.redistribute.even.on.same.rack", true); + conf.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1200000); + conf.set(DFSConfigKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, + ScriptBasedMapping.class.getName()); + conf.set("hbase.coprocessor.master.classes", +// RSGroupMasterObserver.class.getName() + "," + + RSGroupAdminEndpoint.class.getName()); + conf.setInt("hbase.assignment.maximum.attempts", 3); + conf.set("hbase.coprocessor.master.classes", +// RSGroupMasterObserver.class.getName() + "," + + RSGroupAdminEndpoint.class.getName()); + conf.set(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, ""+ 4 ); + conf.setBoolean("hbase.assignment.usezk", false); + conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); + //Making it high value, tests should explicitly call catalog janitor if needed. + conf.setInt("hbase.catalogjanitor.interval", Integer.MAX_VALUE); + //Don't let chore run. + conf.setInt(FavoredNodesRepairChore.FAVORED_NODE_REPAIR_CHORE_FREQ, Integer.MAX_VALUE); + } + + public static class CustomObserver extends BaseRegionObserver { + + @Override + public void postCompleteSplit(ObserverContext ctx) + throws IOException { + postSplit = true; + } + + @Override + public void preRollBackSplit(ObserverContext ctx) + throws IOException { + throw new IOException("failed split roll back"); + } + + @Override + public void preSplitAfterPONR(ObserverContext ctx) + throws IOException { + throw new IOException("failed split"); + } + } + + public static class SplitObserver extends BaseRegionObserver{ + + @Override + public void postCompleteSplit(ObserverContext ctx) + throws IOException { + postSplit = true; + } + } + + @Before + public void startCluster() throws Exception { + TEST_UTIL.startMiniCluster(SLAVES); + TEST_UTIL.getDFSCluster().waitClusterUp(); + admin = TEST_UTIL.getHBaseAdmin(); + //wait for balancer to come online + TEST_UTIL.waitFor(60000*5 , new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return TEST_UTIL.getMiniHBaseCluster().getMaster().isInitialized() + && ((RSGroupBasedLoadBalancer) TEST_UTIL.getMiniHBaseCluster().getMaster() + .getLoadBalancer()).isOnline(); + } + }); + groupAdmin = new RSGroupAdminClient(admin.getConnection()); + admin.setBalancerRunning(false, true); + } + + @After + public void stopCluster() throws Exception { + if (TEST_UTIL.getHBaseClusterInterface() != null) { + for (TableName tName : admin.listTableNames()) { + admin.disableTable(tName); + admin.deleteTable(tName); + } + for (NamespaceDescriptor nspDesc : admin.listNamespaceDescriptors()) { + if (nspDesc.getName().equals(NAMESPACE)) { + admin.deleteNamespace(NAMESPACE); + } + } + removeGroup(groupAdmin, GROUP); + TEST_UTIL.cleanupTestDir(); + TEST_UTIL.shutdownMiniCluster(); + } + } + + @Test + public void testBasicRegionPlacement() throws Exception { + addGroup(groupAdmin, GROUP, 4); + NamespaceDescriptor nsp = NamespaceDescriptor.create(NAMESPACE) + .addConfiguration(RSGroupInfo.NAMESPACEDESC_PROP_GROUP, GROUP).build(); + admin.createNamespace(nsp); + String table = NAMESPACE + TableName.NAMESPACE_DELIM + "testBasicRegionPlacement"; + TableName tableName = TableName.valueOf(table); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM); + FavoredNodesManager fnm = TEST_UTIL.getHBaseCluster().getMaster().getFavoredNodesManager(); + List regionsOfTable = + TEST_UTIL.getHBaseAdmin().getTableRegions(tableName); + Set groupServers = groupAdmin.getRSGroupInfo(GROUP).getServers(); + for (HRegionInfo rInfo : regionsOfTable) { + Set favNodes = Sets.newHashSet(fnm.getFavoredNodes(rInfo)); + assertNotNull(favNodes); + assertEquals(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, favNodes.size()); + for (ServerName sn : favNodes) { + assertTrue("Favored node not in group servers", + groupServers.contains(sn.getHostPort())); + } + } + Map> replicaLoadMap = + fnm.getReplicaLoad(Lists.newArrayList(admin.getClusterStatus().getServers())); + assertTrue("Not all replica load collected.", + admin.getClusterStatus().getServers().size() == replicaLoadMap.size()); + for (Entry> entry : replicaLoadMap.entrySet()) { + assertTrue(entry.getValue().size() == 3); + assertTrue(entry.getValue().get(0) >= 0); + assertTrue(entry.getValue().get(1) >= 0); + assertTrue(entry.getValue().get(2) >= 0); + } + admin.disableTable(tableName); + TEST_UTIL.waitTableDisabled(tableName.getName()); + admin.deleteTable(tableName); + for (HRegionInfo rInfo : regionsOfTable) { + List favNodes = fnm.getFavoredNodes(rInfo); + assertNull(favNodes); + } + replicaLoadMap = + fnm.getReplicaLoad(Lists.newArrayList(admin.getClusterStatus().getServers())); + assertTrue("replica load found " + replicaLoadMap.size() + " instead of 0.", + replicaLoadMap.size() == admin.getClusterStatus().getServers().size()); + } + + @Test + public void testRedistribute() throws Exception { + addGroup(groupAdmin, GROUP, 4); + NamespaceDescriptor nsp = NamespaceDescriptor.create(NAMESPACE) + .addConfiguration(RSGroupInfo.NAMESPACEDESC_PROP_GROUP, GROUP).build(); + admin.createNamespace(nsp); + String tableName = NAMESPACE + TableName.NAMESPACE_DELIM + "testRedistribute"; + HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM); + TEST_UTIL.getHBaseCluster().startRegionServer(); + TEST_UTIL.getHBaseCluster().startRegionServer(); + moveServers(groupAdmin, RSGroupInfo.DEFAULT_GROUP, GROUP, 2); + TEST_UTIL.waitFor(10000, new Predicate() { + @Override + public boolean evaluate() throws Exception { + return groupAdmin.getRSGroupInfo(GROUP).getServers().size()==6; + } + }); + ClusterStatus status = admin.getClusterStatus(); + assertEquals(SLAVES + 2, TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().size()); + // Call redistribute + assertTrue(groupAdmin.redistribute(GROUP)); + assertTrue(TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsInTransition().size() == 0); + // Check the current assignments remain as-is, but the all the region servers should have + // replicas + ClusterStatus newStatus = admin.getClusterStatus(); + for (ServerName sn : status.getServers()) { + assertEquals(status.getLoad(sn).getNumberOfRegions(), newStatus.getLoad(sn) + .getNumberOfRegions()); + } + Map> regionReplicaMap = + getReplicaMap(TableName.valueOf(tableName)); + RSGroupInfo groupInfo = groupAdmin.getRSGroupInfo(GROUP); + int numOnlineGroupServers = groupInfo.getServers().size(); + int minReplicas = REGION_NUM / numOnlineGroupServers; + assertEquals(numOnlineGroupServers, regionReplicaMap.keySet().size()); + checkMinReplicas(regionReplicaMap, minReplicas, groupInfo.getServers()); + + ServerName serverToKill = getGroupServerToKill(groupAdmin, GROUP); + TEST_UTIL.getHBaseCluster().killRegionServer(serverToKill); + TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(serverToKill, -1); + TEST_UTIL.waitFor(1000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return (TEST_UTIL.getHBaseAdmin().getClusterStatus().getServers().size() == SLAVES + 1); + } + }); + TEST_UTIL.waitFor(10000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return (!TEST_UTIL.getHBaseCluster().getMaster().getServerManager() + .areDeadServersInProgress()); + } + }); + TEST_UTIL.waitFor(40000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return (TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates() + .getRegionsInTransition().size() == 0); + } + }); + List deadServersInUse = groupAdmin.checkGroupFavoredNodes(GROUP); + assertNotNull(deadServersInUse); + assertEquals(1, deadServersInUse.size()); + assertTrue(ServerName.isSameHostnameAndPort(deadServersInUse.get(0), serverToKill)); + assertTrue(groupAdmin.redistribute(GROUP)); + assertTrue(TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsInTransition().size() == 0); + regionReplicaMap = getReplicaMap(TableName.valueOf(tableName)); + numOnlineGroupServers = groupInfo.getServers().size() - 1; + minReplicas = REGION_NUM / numOnlineGroupServers; + Set onlineServers = Sets.newHashSet(groupInfo.getServers()); + onlineServers.remove(serverToKill.getHostPort()); + checkMinReplicas(regionReplicaMap, minReplicas, onlineServers); + assertTrue(TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager() + .getRegionStates().getRegionsInTransition().size() == 0); + } + + Map> getReplicaMap(TableName tableName) throws IOException { + List regionsOfTable = TEST_UTIL.getHBaseAdmin().getTableRegions(tableName); + FavoredNodesManager fnm = TEST_UTIL.getHBaseCluster().getMaster().getFavoredNodesManager(); + Map> regionReplicaMap = + new HashMap>(); + for (HRegionInfo rInfo : regionsOfTable) { + List favNodes = fnm.getFavoredNodes(rInfo); + assertNotNull(favNodes); + for (ServerName sn : favNodes) { + List replicas = regionReplicaMap.get(sn); + if (replicas == null) { + replicas = new ArrayList(); + } + replicas.add(rInfo); + regionReplicaMap.put(sn, replicas); + } + } + return regionReplicaMap; + } + + void checkMinReplicas(Map> replicaMap, int minReplicas, + Set servers) { + for (HostAndPort sn : servers) { + assertTrue("Atleast min replica expected.", + replicaMap.get(ServerName.valueOf(sn, ServerName.NON_STARTCODE)).size() >= minReplicas); + } + } + + @Test + public void testCompleteRedistribute() throws Exception { + addGroup(groupAdmin, GROUP, 4); + NamespaceDescriptor nsp = NamespaceDescriptor.create(NAMESPACE) + .addConfiguration(RSGroupInfo.NAMESPACEDESC_PROP_GROUP, GROUP).build(); + admin.createNamespace(nsp); + String tableName = NAMESPACE + TableName.NAMESPACE_DELIM + "testCompleteRedistribute"; + HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM); + TEST_UTIL.waitTableAvailable(TableName.valueOf(tableName)); + assertTrue(groupAdmin.completeRedistribute(GROUP)); + // AM might not have kicked in yet, a small wait should help + Thread.sleep(30 * 1000); + TEST_UTIL.waitUntilAllRegionsAssigned(TableName.valueOf(tableName)); + List tableRegions = admin.getTableRegions(TableName.valueOf(tableName)); + FavoredNodesManager fnm = TEST_UTIL.getHBaseCluster().getMaster().getFavoredNodesManager(); + RegionStates regionStates = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates(); + for (HRegionInfo hri : tableRegions) { + List favoredNodes = fnm.getFavoredNodes(hri); + assertNotNull(favoredNodes); + ServerName current = regionStates.getRegionServerOfRegion(hri); + assertNotNull("Region not hosted on favored nodes." + "Current Server:" + current + " FavNodes:" + favoredNodes, + FavoredNodesPlan.getFavoredServerPosition(favoredNodes, current)); + } + Map> regionReplicaMap = getReplicaMap(TableName.valueOf(tableName)); + RSGroupInfo groupInfo = groupAdmin.getRSGroupInfo(GROUP); + int numOnlineGroupServers = groupInfo.getServers().size(); + assertTrue(numOnlineGroupServers == regionReplicaMap.keySet().size()); + List salList = new ArrayList(); + for (ServerName sn : regionReplicaMap.keySet()) { + salList.add(new ServerAndLoad(sn, regionReplicaMap.get(sn).size())); + } + assertClusterAsBalanced(salList); + } + + @Test + public void testServerMoveReplacesFavNodes() throws Exception { + addGroup(groupAdmin, GROUP, 4); + NamespaceDescriptor nsp = NamespaceDescriptor.create(NAMESPACE) + .addConfiguration(RSGroupInfo.NAMESPACEDESC_PROP_GROUP, GROUP).build(); + admin.createNamespace(nsp); + TableName tableName = TableName.valueOf(NAMESPACE + TableName.NAMESPACE_DELIM + "testServerMoveReplacesFavNodes"); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM); + TEST_UTIL.waitUntilAllRegionsAssigned(tableName); + List deadServersInUse = groupAdmin.checkGroupFavoredNodes(GROUP); + assertNotNull(deadServersInUse); + assertEquals(0, deadServersInUse.size()); + moveServers(groupAdmin, GROUP, RSGroupInfo.DEFAULT_GROUP, 1); + deadServersInUse = groupAdmin.checkGroupFavoredNodes(GROUP); + assertNotNull(deadServersInUse); + assertEquals(0, deadServersInUse.size()); + } + + @Test + public void testMoveTableToGroup() throws Exception { + addGroup(groupAdmin, GROUP, 4); + TableName tableName = TableName.valueOf("testServerMoveReplacesFavNodes"); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM); + RegionPlacementMaintainer rpm = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration()); + String[] table = {tableName.getNameAsString()}; + rpm.setTargetTableName(table); + rpm.moveSecondaryAndTertiaryToNewGroup(GROUP); + Map> tableFavoredNodes = rpm.getFavoredNodesOfTable(tableName); + assertNotNull(tableFavoredNodes); + assertEquals(REGION_NUM, tableFavoredNodes.size()); + List regionsOfTable = + TEST_UTIL.getHBaseAdmin().getTableRegions(tableName); + Set groupServers = groupAdmin.getRSGroupInfo(GROUP).getServers(); + Set defaultServers = groupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers(); + for (HRegionInfo rInfo : regionsOfTable) { + Set favNodes = Sets.newHashSet(tableFavoredNodes.get(rInfo + .getRegionNameAsString())); + assertNotNull(favNodes); + assertEquals(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, favNodes.size()); + int destinationGroupMatchCount = 0; + int currentGroupMatchCount = 0; + for (ServerName sn : favNodes) { + if (defaultServers.contains(sn.getHostPort())) { + currentGroupMatchCount++; + } + if (groupServers.contains(sn.getHostPort())) { + destinationGroupMatchCount++; + } + } + assertEquals(2, destinationGroupMatchCount); + assertEquals(1, currentGroupMatchCount); + HRegionServer currentSn = getRegionServer(TEST_UTIL.getHBaseClusterInterface() + .getServerHoldingRegion(tableName, rInfo.getRegionName())); + assertNotNull(currentSn); + } + groupAdmin.moveTables(Sets.newHashSet(tableName), GROUP); + List deadServersInUse = groupAdmin.checkGroupFavoredNodes(GROUP); + assertNotNull(deadServersInUse); + assertEquals(0, deadServersInUse.size()); + } + + @Test + public void testMoveTableToGroupOnlyTertiary() throws Exception { + addGroup(groupAdmin, GROUP, 4); + TableName tableName = TableName.valueOf("testMoveTableToGroupOnlyTertiary"); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM); + TEST_UTIL.waitUntilAllRegionsAssigned(tableName); + RegionPlacementMaintainer rpm = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration()); + String[] table = {tableName.getNameAsString()}; + rpm.setTargetTableName(table); + Thread.sleep(10000); + List regionsOfTable = + TEST_UTIL.getHBaseAdmin().getTableRegions(tableName); + Map> tableFavoredNodes = rpm.getFavoredNodesOfTable(tableName); + RegionLocator regionLocator = admin.getConnection().getRegionLocator(tableName); + for (HRegionInfo rInfo : regionsOfTable) { + HRegionServer currentSn = getRegionServer(regionLocator.getRegionLocation(rInfo.getStartKey()).getServerName()); + assertNotNull(currentSn); + } + rpm.setTargetTableName(table); + rpm.moveTertiaryToNewGroup(GROUP); + tableFavoredNodes = rpm.getFavoredNodesOfTable(tableName); + assertNotNull(tableFavoredNodes); + assertEquals(REGION_NUM, tableFavoredNodes.size()); + Set groupServers = groupAdmin.getRSGroupInfo(GROUP).getServers(); + Set defaultServers = groupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers(); + verifyFNSpread(tableName, regionsOfTable, tableFavoredNodes, groupServers, defaultServers, 2, 1); + FavoredGroupLoadBalancer balancer = (FavoredGroupLoadBalancer) TEST_UTIL.getHBaseCluster() + .getMaster().getLoadBalancer(); + List onlineGroupServers = FavoredNodeAssignmentHelper + .filterServers(groupAdmin.getRSGroupInfo(GROUP).getServers(), TEST_UTIL.getHBaseCluster() + .getMaster().getServerManager().getOnlineServersList()); + assertTrue(balancer.checkForFavoredNodesInTargetGroup(tableName, GROUP, onlineGroupServers)); + groupAdmin.moveTables(Sets.newHashSet(tableName), GROUP); + List deadServersInUse = groupAdmin.checkGroupFavoredNodes(GROUP); + assertNotNull(deadServersInUse); + assertEquals(0, deadServersInUse.size()); + tableFavoredNodes = rpm.getFavoredNodesOfTable(tableName); + assertNotNull(tableFavoredNodes); + assertEquals(REGION_NUM, tableFavoredNodes.size()); + regionsOfTable = TEST_UTIL.getHBaseAdmin().getTableRegions(tableName); + groupServers = groupAdmin.getRSGroupInfo(GROUP).getServers(); + for (HRegionInfo rInfo : regionsOfTable) { + Set favNodes = Sets.newHashSet(tableFavoredNodes.get(rInfo + .getRegionNameAsString())); + assertNotNull(favNodes); + assertEquals(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, favNodes.size()); + for (ServerName sn : favNodes) { + assertTrue("Favored node not in group servers", + groupServers.contains(sn.getHostAndPort())); + } + } + } + + @Test + public void testFNInheritanceWithoutMove() throws Exception { + addGroup(groupAdmin, GROUP, 4); + Set groupServers = groupAdmin.getRSGroupInfo(GROUP).getServers(); + Set defaultServers = groupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers(); + + final TableName tableName = TableName.valueOf("testFNInheritanceWithoutMove"); + HTableDescriptor desc = new HTableDescriptor(tableName); + HTableInterface hTable = admin.getConnection().getTable(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, Bytes.toBytes("0"), Bytes.toBytes("9"), REGION_NUM); + TEST_UTIL.loadNumericRows(hTable, HConstants.CATALOG_FAMILY, 0, 9999); + TEST_UTIL.waitUntilAllRegionsAssigned(tableName); + RegionPlacementMaintainer rpm = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration()); + String[] table = {tableName.getNameAsString()}; + rpm.setTargetTableName(table); + List regionsOfTable = + TEST_UTIL.getHBaseAdmin().getTableRegions(tableName); + for (HRegionInfo rInfo : regionsOfTable) { + HRegionServer currentSn = getRegionServer(TEST_UTIL.getHBaseClusterInterface() + .getServerHoldingRegion(tableName, rInfo.getRegionName())); + assertNotNull(currentSn); + } + + // Lets get the parent region (which will be split) and its FN + RegionLocator regionLocator = admin.getConnection().getRegionLocator(tableName); + HRegionLocation parentLocation = regionLocator.getRegionLocation(Bytes.toBytes("500")); + // TODO: Check. Using regionLocator instead of the hTable one. +// hTable.getRegionsInRange(Bytes.toBytes("500"), ); + Map> fnBeforeSplit = rpm.getFavoredNodesOfTable(tableName); + String parentRegion = parentLocation.getRegionInfo().getRegionNameAsString(); + List parentFN = fnBeforeSplit.get(parentRegion); + + assertNotNull(fnBeforeSplit); + assertEquals(REGION_NUM, fnBeforeSplit.size()); + assertEquals(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, parentFN.size()); + + admin.split(tableName, Bytes.toBytes("500")); + waitForOnlineRegionCount(admin, tableName, REGION_NUM + 1); + TEST_UTIL.waitUntilAllRegionsAssigned(tableName); + regionsOfTable = + TEST_UTIL.getHBaseAdmin().getTableRegions(tableName); + + Map> fnAfterSplit = rpm.getFavoredNodesOfTable(tableName); + LOG.info(" Regions: " + regionsOfTable); + LOG.info(" Group servers: " + groupServers); + LOG.info(" Default servers: " + defaultServers); + LOG.info(" FN: " + fnAfterSplit); + assertEquals(REGION_NUM + 1, regionsOfTable.size()); + assertNotNull(fnAfterSplit); + assertEquals(REGION_NUM + 1, fnAfterSplit.size()); + verifyFNSpread(tableName, regionsOfTable, fnAfterSplit, groupServers, defaultServers, 3, 0); + + // Lets separate out the daughter region's FN info + Map> daughterFN = new HashMap>(); + daughterFN.putAll(fnAfterSplit); + for (String regionName : fnBeforeSplit.keySet()) { + if (!regionName.equals(parentRegion)) { + daughterFN.remove(regionName); + } + } + assertEquals("Only 2 daughters should be present:" + daughterFN, 2, daughterFN.size()); + String daughter1 = (String) daughterFN.keySet().toArray()[0]; + String daughter2 = (String) daughterFN.keySet().toArray()[1]; + + // Both daughters should have different FN's + assertFalse("daughters should have diff FN's" + daughterFN, + daughterFN.get(daughter1).containsAll(daughterFN.get(daughter2))); + + // Daughters should have atleast 1 FN different from parent + for (String region : daughterFN.keySet()) { + List childFN = daughterFN.get(region); + childFN.removeAll(parentFN); + assertEquals("One RS of child should not belong to parent", 1, childFN.size()); + } + } + + private void waitForOnlineRegionCount(HBaseAdmin admin, TableName tableName, int regions) + throws IOException, InterruptedException { + int attempts = 10; + while (attempts > 0) { + if (admin.getTableRegions(tableName).size() == regions) { + break; + } + LOG.info("Waiting until region count is met"); + Thread.sleep(1000); + attempts--; + } + } + + @Test + public void testFNInheritanceDuringMove() throws Exception { + addGroup(groupAdmin, GROUP, 4); + final TableName tableName = TableName.valueOf("testFNInheritanceDuringMove"); + HTableDescriptor desc = new HTableDescriptor(tableName); + HTableInterface hTable = admin.getConnection().getTable(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, Bytes.toBytes("0"), Bytes.toBytes("9"), REGION_NUM); + TEST_UTIL.loadNumericRows(hTable, HConstants.CATALOG_FAMILY, 0, 9999); + TEST_UTIL.waitUntilAllRegionsAssigned(tableName); + RegionPlacementMaintainer rpm = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration()); + String[] table = {tableName.getNameAsString()}; + rpm.setTargetTableName(table); + List regionsOfTable = + TEST_UTIL.getHBaseAdmin().getTableRegions(tableName); + for (HRegionInfo rInfo : regionsOfTable) { + HRegionServer currentSn = getRegionServer(TEST_UTIL.getHBaseClusterInterface() + .getServerHoldingRegion(tableName, rInfo.getRegionName())); + assertNotNull(currentSn); + } + rpm.moveTertiaryToNewGroup(GROUP); + Thread.sleep(10 * 1000); + Map> tableFavoredNodes = rpm.getFavoredNodesOfTable(tableName); + assertNotNull(tableFavoredNodes); + assertEquals(REGION_NUM, tableFavoredNodes.size()); + + admin.split(tableName, Bytes.toBytes("500")); + waitForOnlineRegionCount(admin, tableName, REGION_NUM + 1); + TEST_UTIL.waitUntilAllRegionsAssigned(tableName); + regionsOfTable = + TEST_UTIL.getHBaseAdmin().getTableRegions(tableName); + Set groupServers = groupAdmin.getRSGroupInfo(GROUP).getServers(); + Set defaultServers = groupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers(); + tableFavoredNodes = rpm.getFavoredNodesOfTable(tableName); + LOG.info(" Regions: " + regionsOfTable); + LOG.info(" Group servers: " + groupServers); + LOG.info(" Default servers: " + defaultServers); + LOG.info(" FN: " + tableFavoredNodes); + verifyFNSpread(tableName, regionsOfTable, tableFavoredNodes, groupServers, defaultServers, 2, 1); + FavoredGroupLoadBalancer balancer = (FavoredGroupLoadBalancer) TEST_UTIL.getHBaseCluster() + .getMaster().getLoadBalancer(); + List onlineGroupServers = FavoredNodeAssignmentHelper + .filterServers(groupAdmin.getRSGroupInfo(GROUP).getServers(), TEST_UTIL.getHBaseCluster() + .getMaster().getServerManager().getOnlineServersList()); + assertTrue(balancer.checkForFavoredNodesInTargetGroup(tableName, GROUP, onlineGroupServers)); + groupAdmin.moveTables(Sets.newHashSet(tableName), GROUP); + List deadServersInUse = groupAdmin.checkGroupFavoredNodes(GROUP); + assertNotNull(deadServersInUse); + assertEquals(deadServersInUse.toString()+ "/" + groupAdmin.getRSGroupInfo(GROUP).getServers(), + 0, deadServersInUse.size()); + tableFavoredNodes = rpm.getFavoredNodesOfTable(tableName); + assertNotNull(tableFavoredNodes); + assertEquals(REGION_NUM + 1, tableFavoredNodes.size()); + regionsOfTable = TEST_UTIL.getHBaseAdmin().getTableRegions(tableName); + groupServers = groupAdmin.getRSGroupInfo(GROUP).getServers(); + for (HRegionInfo rInfo : regionsOfTable) { + Set favNodes = Sets.newHashSet(tableFavoredNodes.get(rInfo.getRegionNameAsString())); + assertNotNull(favNodes); + assertEquals(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, favNodes.size()); + for (ServerName sn : favNodes) { + assertTrue("Favored node not in group servers", + groupServers.contains(sn.getHostPort())); + } + } + } + + private void verifyFNSpread(TableName tableName, List regionsOfTable, + Map> tableFavoredNodes, Set groupServers, + Set defaultServers, int srcMatch, int destMatch) throws IOException { + for (HRegionInfo rInfo : regionsOfTable) { + Set favNodes = Sets.newHashSet(tableFavoredNodes.get(rInfo + .getRegionNameAsString())); + assertNotNull(favNodes); + assertEquals(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, favNodes.size()); + int destinationGroupMatchCount = 0; + int currentGroupMatchCount = 0; + for (ServerName sn : favNodes) { + if (defaultServers.contains(sn.getHostPort())) { + currentGroupMatchCount++; + } + if (groupServers.contains(sn.getHostPort())) { + destinationGroupMatchCount++; + } + } + // Since we moved only tertiary + assertEquals(destMatch, destinationGroupMatchCount); + assertEquals(srcMatch, currentGroupMatchCount); + HRegionServer currentSn = getRegionServer(TEST_UTIL.getHBaseClusterInterface() + .getServerHoldingRegion(tableName, rInfo.getRegionName())); + assertNotNull(currentSn); + } + } + + @Test + public void testMoveTableToGroupWithoutTertiary() throws Exception { + addGroup(groupAdmin, GROUP, 4); + TableName table = TableName.valueOf("testMoveTableToGroupWithoutTertiary"); + HTableDescriptor desc = new HTableDescriptor(table); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM); + RegionPlacementMaintainer rpm = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration()); + Map> tableFavoredNodes = rpm.getFavoredNodesOfTable(table); + assertNotNull(tableFavoredNodes); + assertEquals(REGION_NUM, tableFavoredNodes.size()); + FavoredGroupLoadBalancer balancer = (FavoredGroupLoadBalancer) TEST_UTIL.getHBaseCluster() + .getMaster().getLoadBalancer(); + List onlineGroupServers = FavoredNodeAssignmentHelper + .filterServers(groupAdmin.getRSGroupInfo(GROUP).getServers(), TEST_UTIL.getHBaseCluster() + .getMaster().getServerManager().getOnlineServersList()); + assertFalse(balancer.checkForFavoredNodesInTargetGroup(table, GROUP, onlineGroupServers)); + boolean exception = false; + try { + groupAdmin.moveTables(Sets.newHashSet(table), GROUP); + } catch (IOException exp) { + exception = true; + } + assertTrue("Table was not not supposed to move", exception); + } + + @Test + public void testCreateTableWithLessThanThreeRS() throws Exception { + addGroup(groupAdmin, GROUP, 1); + NamespaceDescriptor nsp = NamespaceDescriptor.create(NAMESPACE) + .addConfiguration(RSGroupInfo.NAMESPACEDESC_PROP_GROUP, GROUP).build(); + admin.createNamespace(nsp); + assertEquals(1, groupAdmin.getRSGroupInfo(GROUP).getServers().size()); + String tableName = NAMESPACE + TableName.NAMESPACE_DELIM + "testCreateTableWithLessThanThreeRS"; + HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + boolean failed = false; + try { + admin.createTable(desc, new byte[][] { Bytes.toBytes("aaa"), Bytes.toBytes("zzz") }); + } catch (IOException exp) { + failed = true; + } finally { + assertTrue(failed); + } + } + + @Test + public void testMoveTableToNewGroupDuringFailedSplit() throws Exception { + String groupName = "testgroup2"; + addGroup(groupAdmin, groupName, 4); + NamespaceDescriptor nsp = NamespaceDescriptor.create(NAMESPACE) + .addConfiguration(RSGroupInfo.NAMESPACEDESC_PROP_GROUP, groupName).build(); + admin.createNamespace(nsp); + TableName tableName = TableName.valueOf(NAMESPACE + TableName.NAMESPACE_DELIM + + "testMoveTableToNewGroupDuringFailedSplit"); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(Bytes.toBytes("info"))); + desc.setConfiguration(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + CustomObserver.class.getName()); + admin.createTable(desc); + HTableInterface htable = admin.getConnection().getTable(tableName); + TEST_UTIL.loadNumericRows(htable, Bytes.toBytes("info"), 1, 1000); + admin.flush(tableName); + postSplit = false; + admin.split(tableName, Bytes.toBytes("500")); + TEST_UTIL.waitFor(10000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return postSplit; + } + }); + TableName tableName2 = TableName.valueOf("testMoveTableToNewGroupDuringFailedSplit2"); + HTableDescriptor desc2 = new HTableDescriptor(tableName2); + desc2.addFamily(new HColumnDescriptor(Bytes.toBytes("info"))); + admin.createTable(desc2); + RegionPlacementMaintainer rpm = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration()); + String[] table = { tableName2.getNameAsString() }; + rpm.setTargetTableName(table); + rpm.moveTertiaryToNewGroup(groupName); + Map> tableFavoredNodes = rpm.getFavoredNodesOfTable(tableName2); + assertNotNull(tableFavoredNodes); + assertEquals(1, tableFavoredNodes.size()); + List regionsOfTable = TEST_UTIL.getHBaseAdmin().getTableRegions(tableName2); + Set groupServers = groupAdmin.getRSGroupInfo(groupName).getServers(); + Set defaultServers = groupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers(); + for (HRegionInfo rInfo : regionsOfTable) { + Set favNodes = Sets.newHashSet(tableFavoredNodes.get(rInfo + .getRegionNameAsString())); + assertNotNull(favNodes); + assertEquals(3, favNodes.size()); + int destinationGroupMatchCount = 0; + int currentGroupMatchCount = 0; + for (ServerName sn : favNodes) { + if (defaultServers.contains(sn.getHostPort())) { + currentGroupMatchCount++; + } + if (groupServers.contains(sn.getHostPort())) { + destinationGroupMatchCount++; + } + } + // Since we moved only tertiary + assertEquals(1, destinationGroupMatchCount); + assertEquals(2, currentGroupMatchCount); + } + // Now move the table to destination group, which has RITs + groupAdmin.moveTables(Sets.newHashSet(tableName2), groupName); + assertTrue(TEST_UTIL.getHBaseCluster().getMaster().isAlive()); + assertEquals(2, admin.listTables().length); + } + + @Test + public void testWholesomeSplit() throws Exception { + addGroup(groupAdmin, GROUP, 4); + NamespaceDescriptor nsp = NamespaceDescriptor.create(NAMESPACE) + .addConfiguration(RSGroupInfo.NAMESPACEDESC_PROP_GROUP, GROUP).build(); + admin.createNamespace(nsp); + TableName tableName = TableName.valueOf(NAMESPACE + TableName.NAMESPACE_DELIM + + "testWholesomedSplit"); + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(Bytes.toBytes("info"))); + desc.setConfiguration(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, + SplitObserver.class.getName()); + admin.createTable(desc); + HTableInterface htable = admin.getConnection().getTable(tableName); + TEST_UTIL.loadNumericRows(htable, Bytes.toBytes("info"), 1, 1000); + admin.flush(tableName); + final int numRegions = admin.getClusterStatus().getRegionsCount(); + postSplit = false; + admin.split(tableName, Bytes.toBytes("500")); + TEST_UTIL.waitFor(60000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return postSplit; + } + }); + TEST_UTIL.waitFor(60000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws Exception { + return admin.getClusterStatus().getRegionsCount() == numRegions + 1; + } + }); + assertEquals(numRegions + 1, admin.getClusterStatus().getRegionsCount()); + TableName tableName2 = TableName.valueOf(NAMESPACE + TableName.NAMESPACE_DELIM + + "testWholesomedSplit2"); + HTableDescriptor desc2 = new HTableDescriptor(tableName2); + desc2.addFamily(new HColumnDescriptor(Bytes.toBytes("info"))); + admin.createTable(desc2); + assertTrue(TEST_UTIL.getHBaseCluster().getMaster().isAlive()); + assertEquals(2, admin.listTables().length); + } + + private RSGroupInfo addGroup(RSGroupAdmin groupAdmin2, String groupName, int serverCount) + throws IOException, InterruptedException { + RSGroupInfo defaultInfo = groupAdmin2.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP); + assertTrue(defaultInfo != null); + assertTrue(defaultInfo.getServers().size() >= serverCount); + groupAdmin2.addRSGroup(groupName); + + Set set = new HashSet(); + for (HostAndPort hostAndPort : defaultInfo.getServers()) { + if (set.size() == serverCount) { + break; + } + set.add(hostAndPort); + } + groupAdmin2.moveServers(set, groupName); + RSGroupInfo result = groupAdmin2.getRSGroupInfo(groupName); + assertTrue(result.getServers().size() >= serverCount); + return result; + } + + private void removeGroup(RSGroupAdmin admin, String groupName) throws IOException { + RSGroupInfo groupInfo = admin.getRSGroupInfo(groupName); + admin.moveTables(groupInfo.getTables(), RSGroupInfo.DEFAULT_GROUP); + admin.moveServers(groupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP); + admin.removeRSGroup(groupName); + } + + private Set moveServers(RSGroupAdmin groupAdmin2, String origin, String destination, int serverCount) + throws IOException { + RSGroupInfo defaultInfo = groupAdmin2.getRSGroupInfo(origin); + assertTrue(defaultInfo != null); + assertTrue(defaultInfo.getServers().size() >= serverCount); + RSGroupInfo oldValue = groupAdmin2.getRSGroupInfo(destination); + Set set = new HashSet(); + for (HostAndPort server : defaultInfo.getServers()) { + if (set.size() == serverCount) { + break; + } + set.add(server); + } + groupAdmin2.moveServers(set, destination); + RSGroupInfo result = groupAdmin2.getRSGroupInfo(destination); + assertTrue(result.getServers().size() == oldValue.getServers().size() + serverCount); + return set; + } + + ServerName getGroupServerToKill(RSGroupAdmin admin, String groupName) throws IOException { + Set servers = admin.getRSGroupInfo(groupName).getServers(); + HostAndPort hostAndPort = servers.iterator().next(); + for (RegionServerThread rs : TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads()){ + ServerName sn = rs.getRegionServer().getServerName(); + if (sn.getHostAndPort().equals(hostAndPort)) { + return sn; + } + } + return null; + } + + HRegionServer getRegionServer(ServerName sn) { + for (RegionServerThread thread : TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()) { + if (thread.getRegionServer().getServerName().equals(sn)) { + return thread.getRegionServer(); + } + } + List thread = TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads(); + Assert.fail("Can't find sn:" + sn + " in list of region servers: " + thread); + return null; + } +} diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRegionPlacement.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRegionPlacement.java new file mode 100644 index 0000000..fc7ff88 --- /dev/null +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRegionPlacement.java @@ -0,0 +1,576 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rsgroup; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper; +import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer; +import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan; +import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan.Position; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.Region; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.net.ScriptBasedMapping; +import org.apache.zookeeper.KeeperException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category({MasterTests.class, MediumTests.class}) +public class TestRegionPlacement { + private static final Log LOG = LogFactory.getLog(TestRegionPlacement.class); + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final static int SLAVES = 10; + private static Connection CONNECTION; + private static Admin admin; + private static RegionPlacementMaintainer rp; + private static Position[] positions = Position.values(); + private int lastRegionOnPrimaryRSCount = 0; + private int REGION_NUM = 10; + private Map favoredNodesAssignmentPlan = + new HashMap(); + + @BeforeClass + public static void setupBeforeClass() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + // Enable the favored nodes based load balancer + conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, + FavoredNodeLoadBalancer.class, LoadBalancer.class); + conf.setBoolean("hbase.tests.use.shortcircuit.reads", false); + conf.set(DFSConfigKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, + ScriptBasedMapping.class.getName()); + TEST_UTIL.startMiniCluster(SLAVES); + TEST_UTIL.getHBaseCluster().waitForActiveAndReadyMaster(); + CONNECTION = TEST_UTIL.getConnection(); + admin = CONNECTION.getAdmin(); + rp = new RegionPlacementMaintainer(conf); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Ignore ("Test for unfinished feature") @Test + public void testRegionPlacement() throws Exception { + String tableStr = "testRegionAssignment"; + TableName table = TableName.valueOf(tableStr); + // Create a table with REGION_NUM regions. + createTable(table, REGION_NUM); + + TEST_UTIL.waitTableAvailable(table); + + // Verify all the user regions are assigned to the primary region server + // based on the plan + verifyRegionOnPrimaryRS(REGION_NUM); + + FavoredNodesPlan currentPlan = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan(); + // Verify all the region server are update with the latest favored nodes + verifyRegionServerUpdated(currentPlan); + // Test Case 2: To verify whether the region placement tools can + // correctly update the new assignment plan to hbase:meta and Region Server. + // The new assignment plan is generated by shuffle the existing assignment + // plan by switching PRIMARY, SECONDARY and TERTIARY nodes. + // Shuffle the plan by switching the secondary region server with + // the tertiary. + + // Shuffle the secondary with tertiary favored nodes + FavoredNodesPlan shuffledPlan = this.shuffleAssignmentPlan(currentPlan, + FavoredNodesPlan.Position.SECONDARY, FavoredNodesPlan.Position.TERTIARY); + + // Let the region placement update the hbase:meta and Region Servers + rp.updateAssignmentPlan(shuffledPlan); + + // Verify the region assignment. There are supposed to no region reassignment + // All the regions are still on the primary region server + verifyRegionAssignment(shuffledPlan, 0, REGION_NUM); + + // Shuffle the plan by switching the primary with secondary and + // verify the region reassignment is consistent with the plan. + shuffledPlan = this.shuffleAssignmentPlan(currentPlan, + FavoredNodesPlan.Position.PRIMARY, FavoredNodesPlan.Position.SECONDARY); + // Update the assignment plan with the balancer, as it no longer reads the META + // for every balance. + updateBalancerWithFavoredNodes(shuffledPlan.getAssignmentMap()); + + // Let the region placement update the hbase:meta and Region Servers + rp.updateAssignmentPlan(shuffledPlan); + + verifyRegionAssignment(shuffledPlan, REGION_NUM, REGION_NUM); + + // also verify that the AssignmentVerificationReport has the correct information + RegionPlacementMaintainer rp = new RegionPlacementMaintainer(TEST_UTIL.getConfiguration()); + // we are interested in only one table (and hence one report) + rp.setTargetTableName(new String[]{tableStr}); + List reports = rp.verifyRegionPlacement(false); + AssignmentVerificationReport report = reports.get(0); + assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0); + assertTrue(report.getNonFavoredAssignedRegions().size() == 0); + assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM); + assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) != 0); + assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) == 0); + assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) == 0); + assertTrue(report.getUnassignedRegions().size() == 0); + // TODO: Do we even need this one? +// assertTrue(serverBefore.containsAll(serverWithPrimary)); + // Check when a RS stops, the regions get assigned to their secondary/tertiary + killRandomServerAndVerifyAssignment(); + + // also verify that the AssignmentVerificationReport has the correct information + reports = rp.verifyRegionPlacement(false); + report = reports.get(0); + assertTrue(report.getRegionsWithoutValidFavoredNodes().size() == 0); + assertTrue(report.getNonFavoredAssignedRegions().size() == 0); + assertTrue(report.getTotalFavoredAssignments() >= REGION_NUM); + assertTrue(report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) > 0); + assertTrue("secondary " + + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) + " tertiary " + + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY), + (report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) > 0 + || report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY) > 0)); + assertTrue((report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.PRIMARY) + + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.SECONDARY) + + report.getNumRegionsOnFavoredNodeByPosition(FavoredNodesPlan.Position.TERTIARY)) == REGION_NUM); + RegionPlacementMaintainer.printAssignmentPlan(currentPlan); + } + + private void updateBalancerWithFavoredNodes(Map> assignmentMap) + throws IOException { + for (Map.Entry> entry : assignmentMap.entrySet()) { + TEST_UTIL.getHBaseCluster().getMaster().getFavoredNodesManager() + .updateFavoredNodes(entry.getKey(), entry.getValue()); + } + } + + private void killRandomServerAndVerifyAssignment() + throws IOException, InterruptedException, KeeperException { + ServerName serverToKill = null; + int killIndex = 0; + Random random = new Random(System.currentTimeMillis()); + ServerName metaServer = TEST_UTIL.getHBaseCluster().getServerHoldingMeta(); + LOG.debug("Server holding meta " + metaServer); + boolean isNamespaceServer = false; + do { + // kill a random non-meta server carrying at least one region + killIndex = random.nextInt(SLAVES); + serverToKill = TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getServerName(); + Collection regs = + TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getOnlineRegionsLocalContext(); + isNamespaceServer = false; + for (Region r : regs) { + if (r.getRegionInfo().getTable().getNamespaceAsString() + .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) { + isNamespaceServer = true; + break; + } + } + } while (ServerName.isSameHostnameAndPort(metaServer, serverToKill) || isNamespaceServer || + TEST_UTIL.getHBaseCluster().getRegionServer(killIndex).getNumberOfOnlineRegions() == 0); + LOG.debug("Stopping RS " + serverToKill); + Map> regionsToVerify = + new HashMap>(); + // mark the regions to track + for (Map.Entry entry : favoredNodesAssignmentPlan.entrySet()) { + ServerName s = entry.getValue()[0]; + if (ServerName.isSameHostnameAndPort(s, serverToKill)) { + regionsToVerify.put(entry.getKey(), new Pair( + entry.getValue()[1], entry.getValue()[2])); + LOG.debug("Adding " + entry.getKey() + " with sedcondary/tertiary " + + entry.getValue()[1] + " " + entry.getValue()[2]); + } + } + int orig = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened(); + TEST_UTIL.getHBaseCluster().stopRegionServer(serverToKill); + TEST_UTIL.getHBaseCluster().waitForRegionServerToStop(serverToKill, 60000); + int curr = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened(); + while (curr - orig < regionsToVerify.size()) { + LOG.debug("Waiting for " + regionsToVerify.size() + " to come online " + + " Current #regions " + curr + " Original #regions " + orig); + Thread.sleep(200); + curr = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getNumRegionsOpened(); + } + // now verify + for (Map.Entry> entry : regionsToVerify.entrySet()) { + ServerName newDestination = TEST_UTIL.getHBaseCluster().getMaster() + .getAssignmentManager().getRegionStates().getRegionServerOfRegion(entry.getKey()); + Pair secondaryTertiaryServers = entry.getValue(); + LOG.debug("New destination for region " + entry.getKey().getEncodedName() + + " " + newDestination +". Secondary/Tertiary are " + secondaryTertiaryServers.getFirst() + + "/" + secondaryTertiaryServers.getSecond()); + if (!(ServerName.isSameHostnameAndPort(newDestination, secondaryTertiaryServers.getFirst())|| + ServerName.isSameHostnameAndPort(newDestination, secondaryTertiaryServers.getSecond()))){ + fail("Region " + entry.getKey() + " not present on any of the expected servers"); + } + } + // start(reinstate) region server since we killed one before + TEST_UTIL.getHBaseCluster().startRegionServer(); + } + + /** + * Used to test the correctness of this class. + */ + @Ignore ("Test for unfinished feature") @Test + public void testRandomizedMatrix() { + int rows = 100; + int cols = 100; + float[][] matrix = new float[rows][cols]; + Random random = new Random(); + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + matrix[i][j] = random.nextFloat(); + } + } + + // Test that inverting a transformed matrix gives the original matrix. + RegionPlacementMaintainer.RandomizedMatrix rm = + new RegionPlacementMaintainer.RandomizedMatrix(rows, cols); + float[][] transformed = rm.transform(matrix); + float[][] invertedTransformed = rm.invert(transformed); + for (int i = 0; i < rows; i++) { + for (int j = 0; j < cols; j++) { + if (matrix[i][j] != invertedTransformed[i][j]) { + throw new RuntimeException(); + } + } + } + + // Test that the indices on a transformed matrix can be inverted to give + // the same values on the original matrix. + int[] transformedIndices = new int[rows]; + for (int i = 0; i < rows; i++) { + transformedIndices[i] = random.nextInt(cols); + } + int[] invertedTransformedIndices = rm.invertIndices(transformedIndices); + float[] transformedValues = new float[rows]; + float[] invertedTransformedValues = new float[rows]; + for (int i = 0; i < rows; i++) { + transformedValues[i] = transformed[i][transformedIndices[i]]; + invertedTransformedValues[i] = matrix[i][invertedTransformedIndices[i]]; + } + Arrays.sort(transformedValues); + Arrays.sort(invertedTransformedValues); + if (!Arrays.equals(transformedValues, invertedTransformedValues)) { + throw new RuntimeException(); + } + } + + /** + * Shuffle the assignment plan by switching two favored node positions. + * @param plan The assignment plan + * @param p1 The first switch position + * @param p2 The second switch position + * @return + */ + private FavoredNodesPlan shuffleAssignmentPlan(FavoredNodesPlan plan, + FavoredNodesPlan.Position p1, FavoredNodesPlan.Position p2) { + FavoredNodesPlan shuffledPlan = new FavoredNodesPlan(); + + for (Map.Entry> entry : + plan.getAssignmentMap().entrySet()) { + HRegionInfo region = entry.getKey(); + + // copy the server list from the original plan + List shuffledServerList = new ArrayList(); + shuffledServerList.addAll(entry.getValue()); + + // start to shuffle + shuffledServerList.set(p1.ordinal(), entry.getValue().get(p2.ordinal())); + shuffledServerList.set(p2.ordinal(), entry.getValue().get(p1.ordinal())); + + // update the plan + shuffledPlan.updateAssignmentPlan(region, shuffledServerList); + } + return shuffledPlan; + } + + /** + * To verify the region assignment status. + * It will check the assignment plan consistency between hbase:meta and + * region servers. + * Also it will verify weather the number of region movement and + * the number regions on the primary region server are expected + * + * @param plan + * @param regionMovementNum + * @param numRegionsOnPrimaryRS + * @throws InterruptedException + * @throws IOException + */ + private void verifyRegionAssignment(FavoredNodesPlan plan, + int regionMovementNum, int numRegionsOnPrimaryRS) + throws InterruptedException, IOException { + // Verify the assignment plan in hbase:meta is consistent with the expected plan. + verifyMETAUpdated(plan); + + // Verify the number of region movement is expected + verifyRegionMovementNum(regionMovementNum); + + // Verify the number of regions is assigned to the primary region server + // based on the plan is expected + verifyRegionOnPrimaryRS(numRegionsOnPrimaryRS); + + // Verify all the online region server are updated with the assignment plan + verifyRegionServerUpdated(plan); + } + + /** + * Verify the meta has updated to the latest assignment plan + * @param plan + * @throws IOException + */ + private void verifyMETAUpdated(FavoredNodesPlan expectedPlan) throws IOException { + FavoredNodesPlan planFromMETA = rp.getRegionAssignmentSnapshot().getExistingAssignmentPlan(); + assertTrue("The assignment plan is NOT consistent with the expected plan ", + planFromMETA.equals(expectedPlan)); + } + + /** + * Verify the number of region movement is expected + */ + private void verifyRegionMovementNum(int expected) + throws InterruptedException, IOException { + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + HMaster m = cluster.getMaster(); + int lastRegionOpenedCount = m.getAssignmentManager().getNumRegionsOpened(); + // get the assignments start to execute + m.balance(); + + int retry = 10; + long sleep = 3000; + int attempt = 0; + int currentRegionOpened, regionMovement; + do { + currentRegionOpened = m.getAssignmentManager().getNumRegionsOpened(); + regionMovement= currentRegionOpened - lastRegionOpenedCount; + LOG.debug("There are " + regionMovement + "/" + expected + + " regions moved after " + attempt + " attempts"); + Thread.sleep((++attempt) * sleep); + } while (regionMovement != expected && attempt <= retry); + + // update the lastRegionOpenedCount + lastRegionOpenedCount = currentRegionOpened; + + assertEquals("There are only " + regionMovement + " instead of " + + expected + " region movement for " + attempt + " attempts", + regionMovement, expected); + } + + /** + * Verify the number of user regions is assigned to the primary + * region server based on the plan is expected + * @param expectedNum. + * @throws IOException + */ + private void verifyRegionOnPrimaryRS(int expectedNum) + throws IOException { + lastRegionOnPrimaryRSCount = getNumRegionisOnPrimaryRS(); + assertEquals("Only " + expectedNum + " of user regions running " + + "on the primary region server", expectedNum , + lastRegionOnPrimaryRSCount); + } + + /** + * Verify all the online region servers has been updated to the + * latest assignment plan + * @param plan + * @throws IOException + */ + private void verifyRegionServerUpdated(FavoredNodesPlan plan) throws IOException { + // Verify all region servers contain the correct favored nodes information + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + for (int i = 0; i < SLAVES; i++) { + HRegionServer rs = cluster.getRegionServer(i); + for (Region region: rs.getOnlineRegions(TableName.valueOf("testRegionAssignment"))) { + InetSocketAddress[] favoredSocketAddress = rs.getFavoredNodesForRegion( + region.getRegionInfo().getEncodedName()); + List favoredServerList = plan.getAssignmentMap().get(region.getRegionInfo()); + + // All regions are supposed to have favored nodes, + // except for hbase:meta and ROOT + if (favoredServerList == null) { + HTableDescriptor desc = region.getTableDesc(); + // Verify they are ROOT and hbase:meta regions since no favored nodes + assertNull(favoredSocketAddress); + assertTrue("User region " + + region.getTableDesc().getTableName() + + " should have favored nodes", + (desc.isRootRegion() || desc.isMetaRegion())); + } else { + // For user region, the favored nodes in the region server should be + // identical to favored nodes in the assignmentPlan + assertTrue(favoredSocketAddress.length == favoredServerList.size()); + assertTrue(favoredServerList.size() > 0); + for (int j = 0; j < favoredServerList.size(); j++) { + InetSocketAddress addrFromRS = favoredSocketAddress[j]; + InetSocketAddress addrFromPlan = InetSocketAddress.createUnresolved( + favoredServerList.get(j).getHostname(), favoredServerList.get(j).getPort()); + + assertNotNull(addrFromRS); + assertNotNull(addrFromPlan); + assertTrue("Region server " + rs.getServerName().getHostAndPort() + + " has the " + positions[j] + + " for region " + region.getRegionInfo().getRegionNameAsString() + " is " + + addrFromRS + " which is inconsistent with the plan " + + addrFromPlan, addrFromRS.equals(addrFromPlan)); + } + } + } + } + } + + /** + * Check whether regions are assigned to servers consistent with the explicit + * hints that are persisted in the hbase:meta table. + * Also keep track of the number of the regions are assigned to the + * primary region server. + * @return the number of regions are assigned to the primary region server + * @throws IOException + */ + private int getNumRegionisOnPrimaryRS() throws IOException { + final AtomicInteger regionOnPrimaryNum = new AtomicInteger(0); + final AtomicInteger totalRegionNum = new AtomicInteger(0); + LOG.info("The start of region placement verification"); + MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() { + public boolean visit(Result result) throws IOException { + try { + @SuppressWarnings("deprecation") + HRegionInfo info = MetaTableAccessor.getHRegionInfo(result); + if(info.getTable().getNamespaceAsString() + .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) { + return true; + } + byte[] server = result.getValue(HConstants.CATALOG_FAMILY, + HConstants.SERVER_QUALIFIER); + byte[] favoredNodes = result.getValue(HConstants.CATALOG_FAMILY, + FavoredNodeAssignmentHelper.FAVOREDNODES_QUALIFIER); + // Add the favored nodes into assignment plan + ServerName[] favoredServerList = + FavoredNodeAssignmentHelper.getFavoredNodesList(favoredNodes); + favoredNodesAssignmentPlan.put(info, favoredServerList); + + Position[] positions = Position.values(); + if (info != null) { + totalRegionNum.incrementAndGet(); + if (server != null) { + ServerName serverName = + ServerName.valueOf(Bytes.toString(server), -1); + if (favoredNodes != null) { + String placement = "[NOT FAVORED NODE]"; + for (int i = 0; i < favoredServerList.length; i++) { + if (favoredServerList[i].equals(serverName)) { + placement = positions[i].toString(); + if (i == Position.PRIMARY.ordinal()) { + regionOnPrimaryNum.incrementAndGet(); + } + break; + } + } + LOG.info(info.getRegionNameAsString() + " on " + + serverName + " " + placement); + } else { + LOG.info(info.getRegionNameAsString() + " running on " + + serverName + " but there is no favored region server"); + } + } else { + LOG.info(info.getRegionNameAsString() + + " not assigned to any server"); + } + } + return true; + } catch (RuntimeException e) { + LOG.error("Result=" + result); + throw e; + } + } + }; + MetaTableAccessor.fullScanRegions(CONNECTION, visitor); + LOG.info("There are " + regionOnPrimaryNum.intValue() + " out of " + + totalRegionNum.intValue() + " regions running on the primary" + + " region servers" ); + return regionOnPrimaryNum.intValue() ; + } + + /** + * Create a table with specified table name and region number. + * @param tablename + * @param regionNum + * @return + * @throws IOException + */ + private static void createTable(TableName tableName, int regionNum) + throws IOException { + int expectedRegions = regionNum; + byte[][] splitKeys = new byte[expectedRegions - 1][]; + for (int i = 1; i < expectedRegions; i++) { + byte splitKey = (byte) i; + splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey }; + } + + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); + admin.createTable(desc, splitKeys); + + try (RegionLocator r = CONNECTION.getRegionLocator(tableName)) { + List regions = r.getAllRegionLocations(); + assertEquals("Tried to create " + expectedRegions + " regions " + + "but only found " + regions.size(), expectedRegions, regions.size()); + } + } +} diff --git a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java index d1f4898..ef940b1 100644 --- a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java +++ b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdminClient.java @@ -23,6 +23,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.net.HostAndPort; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Result; @@ -105,6 +106,18 @@ public class VerifyingRSGroupAdminClient extends RSGroupAdmin { return wrapped.getRSGroupOfServer(hostPort); } + @Override public boolean redistribute(String name) throws IOException { + return false; + } + + @Override public boolean completeRedistribute(String name) throws IOException { + return false; + } + + @Override public List checkGroupFavoredNodes(String groupName) throws IOException { + return null; + } + public void verify() throws IOException { Map groupMap = Maps.newHashMap(); Set zList = Sets.newHashSet(); diff --git a/hbase-shell/src/main/ruby/hbase/rsgroup_admin.rb b/hbase-shell/src/main/ruby/hbase/rsgroup_admin.rb index 51a4efb..19f7c07 100644 --- a/hbase-shell/src/main/ruby/hbase/rsgroup_admin.rb +++ b/hbase-shell/src/main/ruby/hbase/rsgroup_admin.rb @@ -146,5 +146,23 @@ module Hbase res end + #---------------------------------------------------------------------------------------------- + # redistribute a group + def redistribute(group_name) + @admin.redistribute(group_name) + end + + #---------------------------------------------------------------------------------------------- + # complete redistribute a group + def completeRedistribute(group_name) + @admin.completeRedistribute(group_name) + end + + #---------------------------------------------------------------------------------------------- + # Scans all the favored nodes of regions to look for dead servers being used. + def checkFavoredNodes(group_name) + @admin.checkGroupFavoredNodes(group_name).map { |s| s.getHostAndPort } + end + end end diff --git a/hbase-shell/src/main/ruby/shell/commands/check_group_favored_nodes.rb b/hbase-shell/src/main/ruby/shell/commands/check_group_favored_nodes.rb new file mode 100644 index 0000000..db8391f --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/check_group_favored_nodes.rb @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +module Shell + module Commands + class CheckGroupFavoredNodes < Command + def help + return <<-EOF +Scans all the regions to see if all the favored nodes used by the region are online +region servers. This command returns a list of dead servers being referenced. + hbase> check_group_favored_nodes 'mygroup' +EOF + end + def command(group_name) + list = group_admin.checkFavoredNodes(group_name) + list.each do |server| + formatter.row([ server ]) + end + end + end + end +end diff --git a/hbase-shell/src/main/ruby/shell/commands/complete_redistribute_group.rb b/hbase-shell/src/main/ruby/shell/commands/complete_redistribute_group.rb new file mode 100644 index 0000000..fdbdc4d --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/complete_redistribute_group.rb @@ -0,0 +1,37 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class CompleteRedistributeGroup < Command + def help + return <<-EOF +Trigger the balancer for completely redistributing favored nodes. All the favored nodes will be +redistributed.. Please note that this command will not move any regions from the region servers. + hbase> complete_redistribute_group 'my_group' +EOF + end + + def command(group_name) + group_admin.completeRedistribute(group_name) + end + end + end +end diff --git a/hbase-shell/src/main/ruby/shell/commands/redistribute_group.rb b/hbase-shell/src/main/ruby/shell/commands/redistribute_group.rb new file mode 100644 index 0000000..2271bc4 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/redistribute_group.rb @@ -0,0 +1,38 @@ +# +# Copyright The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class RedistributeGroup < Command + def help + return <<-EOF +Trigger the balancer for redistributing favored nodes. Returns true if balancer ran and was able to +update the meta with the new favored node information. Please note that this balance action will not +move any regions from the region servers. + hbase> redistribute_group 'my_group' +EOF + end + + def command(group_name) + group_admin.redistribute(group_name) + end + end + end +end