From f806b86d8fa94baac20cd476f97d04ab4da29779 Mon Sep 17 00:00:00 2001 From: zhangduo Date: Wed, 19 Feb 2020 17:26:02 -0800 Subject: [PATCH] HBASE-23878 Backport HBASE-22040 HBASE-22040 - Add mergeRegionsAsync with a List of region names method in AsyncAdmin Signed-off-by: Zheng Hu Signed-off-by: Sakthi (cherry picked from commit 8a5101af2186443ae919805933a5fbb4085b1468) --- .../org/apache/hadoop/hbase/client/Admin.java | 28 ++++++----- .../hadoop/hbase/client/AsyncAdmin.java | 19 +++++++- .../hadoop/hbase/client/AsyncHBaseAdmin.java | 5 +- .../hadoop/hbase/client/HBaseAdmin.java | 37 ++++---------- .../hbase/client/RawAsyncHBaseAdmin.java | 48 +++++++++---------- .../hbase/master/MasterRpcServices.java | 5 +- .../hadoop/hbase/client/TestAdmin1.java | 46 ++++++++++++++++++ .../client/TestAsyncRegionAdminApi2.java | 41 ++++++++++++++++ 8 files changed, 158 insertions(+), 71 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index d17b709be4ff6d4067988e5fc6337abfbd5927b4..66912bfbac9d65fb1cf7ff2202040370a2a031a5 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -1282,29 +1282,31 @@ public interface Admin extends Abortable, Closeable { /** * Merge two regions. Asynchronous operation. - * * @param nameOfRegionA encoded or full name of region a * @param nameOfRegionB encoded or full name of region b - * @param forcible true if do a compulsory merge, otherwise we will only merge - * two adjacent regions - * @throws IOException + * @param forcible true if do a compulsory merge, otherwise we will only merge two + * adjacent regions */ - Future mergeRegionsAsync( - byte[] nameOfRegionA, - byte[] nameOfRegionB, - boolean forcible) throws IOException; + default Future mergeRegionsAsync(byte[] nameOfRegionA, byte[] nameOfRegionB, + boolean forcible) throws IOException { + byte[][] nameofRegionsToMerge = new byte[2][]; + nameofRegionsToMerge[0] = nameOfRegionA; + nameofRegionsToMerge[1] = nameOfRegionB; + return mergeRegionsAsync(nameofRegionsToMerge, forcible); + } /** * Merge regions. Asynchronous operation. - * + *

+ * You may get a {@code DoNotRetryIOException} if you pass more than two regions in but the master + * does not support merging more than two regions. At least till 2.2.0, we still only support + * merging two regions. * @param nameofRegionsToMerge encoded or full name of daughter regions * @param forcible true if do a compulsory merge, otherwise we will only merge * adjacent regions - * @throws IOException */ - Future mergeRegionsAsync( - byte[][] nameofRegionsToMerge, - boolean forcible) throws IOException; + Future mergeRegionsAsync(byte[][] nameofRegionsToMerge, boolean forcible) + throws IOException; /** + Split a table. The method will execute split action for each region in table. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index 84bf3c15116e5a58b13963fa7f27398d4f0fbe99..a66979058cddd6b2ca38f91d1e71765ac2cb4c76 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.client; import com.google.protobuf.RpcChannel; +import java.util.Arrays; import java.util.Collection; import java.util.EnumSet; import java.util.List; @@ -474,8 +475,22 @@ public interface AsyncAdmin { * @param forcible true if do a compulsory merge, otherwise we will only merge two adjacent * regions */ - CompletableFuture mergeRegions(byte[] nameOfRegionA, byte[] nameOfRegionB, - boolean forcible); + default CompletableFuture mergeRegions(byte[] nameOfRegionA, byte[] nameOfRegionB, + boolean forcible) { + return mergeRegions(Arrays.asList(nameOfRegionA, nameOfRegionB), forcible); + } + + /** + * Merge regions. + *

+ * You may get a {@code DoNotRetryIOException} if you pass more than two regions in but the master + * does not support merging more than two regions. At least till 2.2.0, we still only support + * merging two regions. + * @param nameOfRegionsToMerge encoded or full name of daughter regions + * @param forcible true if do a compulsory merge, otherwise we will only merge two adjacent + * regions + */ + CompletableFuture mergeRegions(List nameOfRegionsToMerge, boolean forcible); /** * Split a table. The method will execute split action for each region in table. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index 8258b032e639e41827d4bc33e51d538592075e7e..e8e3e04a7b3f2dbacee39ac592dada270afe7a16 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -313,9 +313,8 @@ class AsyncHBaseAdmin implements AsyncAdmin { } @Override - public CompletableFuture mergeRegions(byte[] nameOfRegionA, byte[] nameOfRegionB, - boolean forcible) { - return wrap(rawAdmin.mergeRegions(nameOfRegionA, nameOfRegionB, forcible)); + public CompletableFuture mergeRegions(List nameOfRegionsToMerge, boolean forcible) { + return wrap(rawAdmin.mergeRegions(nameOfRegionsToMerge, forcible)); } @Override diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 109a66fd7a07a6fdc86ab7f9c5ba4e907b9852ba..5db75b6447a0199611e54567dc437459b54c7cff 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -107,6 +107,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; +import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; @@ -1677,42 +1678,22 @@ public class HBaseAdmin implements Admin { mergeRegionsAsync(nameOfRegionA, nameOfRegionB, forcible); } - /** - * Merge two regions. Asynchronous operation. - * @param nameOfRegionA encoded or full name of region a - * @param nameOfRegionB encoded or full name of region b - * @param forcible true if do a compulsory merge, otherwise we will only merge - * two adjacent regions - * @throws IOException - */ - @Override - public Future mergeRegionsAsync( - final byte[] nameOfRegionA, - final byte[] nameOfRegionB, - final boolean forcible) throws IOException { - byte[][] nameofRegionsToMerge = new byte[2][]; - nameofRegionsToMerge[0] = nameOfRegionA; - nameofRegionsToMerge[1] = nameOfRegionB; - return mergeRegionsAsync(nameofRegionsToMerge, forcible); - } - /** * Merge two regions. Asynchronous operation. * @param nameofRegionsToMerge encoded or full name of daughter regions * @param forcible true if do a compulsory merge, otherwise we will only merge * adjacent regions - * @throws IOException */ @Override - public Future mergeRegionsAsync( - final byte[][] nameofRegionsToMerge, - final boolean forcible) throws IOException { - assert(nameofRegionsToMerge.length >= 2); + public Future mergeRegionsAsync(final byte[][] nameofRegionsToMerge, final boolean forcible) + throws IOException { + Preconditions.checkArgument(nameofRegionsToMerge.length >= 2, "Can not merge only %s region", + nameofRegionsToMerge.length); byte[][] encodedNameofRegionsToMerge = new byte[nameofRegionsToMerge.length][]; - for(int i = 0; i < nameofRegionsToMerge.length; i++) { - encodedNameofRegionsToMerge[i] = HRegionInfo.isEncodedRegionName(nameofRegionsToMerge[i]) ? - nameofRegionsToMerge[i] : - Bytes.toBytes(HRegionInfo.encodeRegionName(nameofRegionsToMerge[i])); + for (int i = 0; i < nameofRegionsToMerge.length; i++) { + encodedNameofRegionsToMerge[i] = + RegionInfo.isEncodedRegionName(nameofRegionsToMerge[i]) ? nameofRegionsToMerge[i] + : Bytes.toBytes(RegionInfo.encodeRegionName(nameofRegionsToMerge[i])); } TableName tableName = null; diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index aae0101b436185b19842baaf20948b9e1447bb4b..059c045ce1f2bcb2fefb1a2b5434f5807bd43df6 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -1129,13 +1129,12 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { }); } - private CompletableFuture checkRegionsAndGetTableName(byte[] encodeRegionNameA, - byte[] encodeRegionNameB) { + private CompletableFuture checkRegionsAndGetTableName(byte[][] encodedRegionNames) { AtomicReference tableNameRef = new AtomicReference<>(); CompletableFuture future = new CompletableFuture<>(); - - checkAndGetTableName(encodeRegionNameA, tableNameRef, future); - checkAndGetTableName(encodeRegionNameB, tableNameRef, future); + for (byte[] encodedRegionName : encodedRegionNames) { + checkAndGetTableName(encodedRegionName, tableNameRef, future); + } return future; } @@ -1185,28 +1184,29 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { } @Override - public CompletableFuture mergeRegions(byte[] nameOfRegionA, byte[] nameOfRegionB, - boolean forcible) { + public CompletableFuture mergeRegions(List nameOfRegionsToMerge, boolean forcible) { + if (nameOfRegionsToMerge.size() < 2) { + return failedFuture(new IllegalArgumentException( + "Can not merge only " + nameOfRegionsToMerge.size() + " region")); + } CompletableFuture future = new CompletableFuture<>(); - final byte[] encodeRegionNameA = toEncodeRegionName(nameOfRegionA); - final byte[] encodeRegionNameB = toEncodeRegionName(nameOfRegionB); + byte[][] encodedNameOfRegionsToMerge = + nameOfRegionsToMerge.stream().map(this::toEncodeRegionName).toArray(byte[][]::new); - addListener(checkRegionsAndGetTableName(encodeRegionNameA, encodeRegionNameB), - (tableName, err) -> { - if (err != null) { - future.completeExceptionally(err); - return; - } + addListener(checkRegionsAndGetTableName(encodedNameOfRegionsToMerge), (tableName, err) -> { + if (err != null) { + future.completeExceptionally(err); + return; + } - MergeTableRegionsRequest request = null; - try { - request = RequestConverter.buildMergeTableRegionsRequest( - new byte[][] { encodeRegionNameA, encodeRegionNameB }, forcible, ng.getNonceGroup(), - ng.newNonce()); - } catch (DeserializationException e) { - future.completeExceptionally(e); - return; - } + MergeTableRegionsRequest request = null; + try { + request = RequestConverter.buildMergeTableRegionsRequest(encodedNameOfRegionsToMerge, + forcible, ng.getNonceGroup(), ng.newNonce()); + } catch (DeserializationException e) { + future.completeExceptionally(e); + return; + } addListener( this. procedureCall(request, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 949aed5570b4ea6e2c5f5d57ce845cdbc5d81b82..18ede393b10333fe08c6db4e5e122bd34da05f61 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -784,7 +784,10 @@ public class MasterRpcServices extends RSRpcServices RegionStates regionStates = master.getAssignmentManager().getRegionStates(); - assert(request.getRegionCount() == 2); + if (request.getRegionCount() != 2) { + throw new ServiceException(new DoNotRetryIOException( + "Only support merging 2 regions but " + request.getRegionCount() + " region passed")); + } RegionInfo[] regionsToMerge = new RegionInfo[request.getRegionCount()]; for (int i = 0; i < request.getRegionCount(); i++) { final byte[] encodedNameOfRegion = request.getRegion(i).getValue().toByteArray(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java index d4a662d5cae0d2dd703c771e441d891b8a1c23ab..17788dcf0d03dc799148ee913ede9bfc422b97ab 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java @@ -29,8 +29,10 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; @@ -1436,6 +1438,50 @@ public class TestAdmin1 { } } + @Test + public void testMergeRegionsInvalidRegionCount() + throws IOException, InterruptedException, ExecutionException { + TableName tableName = TableName.valueOf(name.getMethodName()); + TableDescriptor td = TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of("d")).build(); + byte[][] splitRows = new byte[2][]; + splitRows[0] = new byte[] { (byte) '3' }; + splitRows[1] = new byte[] { (byte) '6' }; + try { + TEST_UTIL.createTable(td, splitRows); + TEST_UTIL.waitTableAvailable(tableName); + + List tableRegions = admin.getRegions(tableName); + // 0 + try { + admin.mergeRegionsAsync(new byte[0][0], false).get(); + fail(); + } catch (IllegalArgumentException e) { + // expected + } + // 1 + try { + admin.mergeRegionsAsync(new byte[][] { tableRegions.get(0).getEncodedNameAsBytes() }, false) + .get(); + fail(); + } catch (IllegalArgumentException e) { + // expected + } + // 3 + try { + admin.mergeRegionsAsync( + tableRegions.stream().map(RegionInfo::getEncodedNameAsBytes).toArray(byte[][]::new), + false).get(); + fail(); + } catch (DoNotRetryIOException e) { + // expected + } + } finally { + admin.disableTable(tableName); + admin.deleteTable(tableName); + } + } + @Test public void testSplitShouldNotHappenIfSplitIsDisabledForTable() throws Exception { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi2.java index 89f7b4a397bcecad8236e688e0fd0d2e74b6facf..830ffecba0b608c668173300b091e8431c151f7f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi2.java @@ -18,13 +18,20 @@ package org.apache.hadoop.hbase.client; import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME; +import static org.hamcrest.CoreMatchers.instanceOf; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import org.apache.hadoop.hbase.AsyncMetaTableAccessor; +import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionLocation; @@ -175,6 +182,40 @@ public class TestAsyncRegionAdminApi2 extends TestAsyncAdminBase { assertEquals(1, regionLocations.size()); } + @Test + public void testMergeRegionsInvalidRegionCount() throws InterruptedException { + byte[][] splitRows = new byte[][] { Bytes.toBytes("3"), Bytes.toBytes("6") }; + createTableWithDefaultConf(tableName, splitRows); + List regions = admin.getRegions(tableName).join(); + // 0 + try { + admin.mergeRegions(Collections.emptyList(), false).get(); + fail(); + } catch (ExecutionException e) { + // expected + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + } + // 1 + try { + admin.mergeRegions(regions.stream().limit(1).map(RegionInfo::getEncodedNameAsBytes) + .collect(Collectors.toList()), false).get(); + fail(); + } catch (ExecutionException e) { + // expected + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + } + // 3 + try { + admin.mergeRegions( + regions.stream().map(RegionInfo::getEncodedNameAsBytes).collect(Collectors.toList()), false) + .get(); + fail(); + } catch (ExecutionException e) { + // expected + assertThat(e.getCause(), instanceOf(DoNotRetryIOException.class)); + } + } + @Test public void testSplitTable() throws Exception { initSplitMergeSwitch(); -- 2.20.1 (Apple Git-117)