From dc0fc6811cdf58964f0088f359e7f86dda9d8f5b Mon Sep 17 00:00:00 2001 From: wangyuan Date: Tue, 2 May 2017 16:06:58 +0800 Subject: [PATCH] HBASE-17983 control region numbers when create table to improve performance --- .../hadoop/hbase/RegionTooManyException.java | 44 ++++++++++++++++++++++ .../org/apache/hadoop/hbase/client/HBaseAdmin.java | 11 ++++++ .../java/org/apache/hadoop/hbase/HConstants.java | 10 +++++ hbase-common/src/main/resources/hbase-default.xml | 7 ++++ 4 files changed, 72 insertions(+) create mode 100644 hbase-client/src/main/java/org/apache/hadoop/hbase/RegionTooManyException.java diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionTooManyException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionTooManyException.java new file mode 100644 index 0000000..06bd8e1 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/RegionTooManyException.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +/** + * Thrown by a region server if region numbers is too many . + */ +@InterfaceAudience.Public +public class RegionTooManyException extends IOException { + private static final long serialVersionUID = 1728345723728342L; + + /** default constructor */ + public RegionTooManyException() { + super(); + } + + /** + * Constructor + * @param msg message + */ + public RegionTooManyException(final String msg) { + super(msg); + } +} + diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 7e90ff3..16bdf2e 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -245,6 +245,8 @@ public class HBaseAdmin implements Admin { private NonceGenerator ng; + private int avgRegionLoad; + @Override public int getOperationTimeout() { return operationTimeout; @@ -270,6 +272,9 @@ public class HBaseAdmin implements Admin { this.rpcControllerFactory = connection.getRpcControllerFactory(); this.ng = this.connection.getNonceGenerator(); + + this.avgRegionLoad = this.conf.getInt(HConstants.HBASE_CLIENT_REGION_AVERAGELOAD_NUMBERS, + HConstants.DEFAULT_HBASE_CLIENT_REGION_AVERAGELOAD_NUMBERS); } @Override @@ -492,6 +497,12 @@ public class HBaseAdmin implements Admin { if (desc.getTableName() == null) { throw new IllegalArgumentException("TableName cannot be null"); } + + double averageLoad = this.getClusterStatus().getAverageLoad(); + if (averageLoad > this.avgRegionLoad){ + throw new RegionTooManyException("Region averageLoad must less than configuration."); + } + if (splitKeys != null && splitKeys.length > 0) { Arrays.sort(splitKeys, Bytes.BYTES_COMPARATOR); // Verify there are no duplicate split keys diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 3c1b021..d35cca5 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -764,6 +764,16 @@ public final class HConstants { public static final int DEFAULT_HBASE_CLIENT_MAX_PERREGION_TASKS = 1; /** + * The maximum number of every regionserver region average load. + */ + public static final String HBASE_CLIENT_REGION_AVERAGELOAD_NUMBERS = "hbase.client.region.averageload.numbers"; + + /** + * Default value of {@link #HBASE_CLIENT_REGION_AVERAGELOAD_NUMBERS}. + */ + public static final int DEFAULT_HBASE_CLIENT_REGION_AVERAGELOAD_NUMBERS = 500; + + /** * The maximum number of concurrent pending RPC requests for one server in process level. */ public static final String HBASE_CLIENT_PERSERVER_REQUESTS_THRESHOLD = diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index 40cf675..11baa9a 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -541,6 +541,13 @@ possible configurations would overwhelm and obscure the important. 2 + + hbase.client.region.averageload.numbers + 500 + + + + hbase.bulkload.retries.number -- 1.9.1