From f0c97f125f715bd976532f0e7de1ba0aabd87a93 Mon Sep 17 00:00:00 2001 From: Sakthi Date: Thu, 13 Feb 2020 14:40:46 -0800 Subject: [PATCH] HBASE-22827 Expose multi-region merge in shell and Admin API mergeRegionsAsync admin API with two regions as parameters deprecated since 2.3.0 and removed from 4.0 merge_region shell command now supports multiple regions merge since 2.3.0 & 3.0.0 Signed-off-by: Viraj Jasani Signed-off-by: Esteban Gutierrez Signed-off-by: Josh Elser (cherry picked from commit 5f61df479260598ea8f7856b4abc2541c27ed5db) --- .../org/apache/hadoop/hbase/client/Admin.java | 9 ++- .../hadoop/hbase/client/AsyncAdmin.java | 9 ++- hbase-shell/src/main/ruby/hbase/admin.rb | 28 ++++++++-- hbase-shell/src/main/ruby/shell/commands.rb | 6 +- .../main/ruby/shell/commands/merge_region.rb | 34 ++++++++++-- hbase-shell/src/test/ruby/hbase/admin_test.rb | 55 +++++++++++++++++++ 6 files changed, 120 insertions(+), 21 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index f77be3357306400e4805b16ec3ec5ffe60fde725..a8029f8cba99ee07ab5f3b3e0106652523a0fb42 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -1413,7 +1413,10 @@ public interface Admin extends Abortable, Closeable { * @param forcible true if do a compulsory merge, otherwise we will only merge two * adjacent regions * @throws IOException if a remote or network exception occurs + * @deprecated since 2.3.0 and will be removed in 4.0.0. Multi-region merge feature is now + * supported. Use {@link #mergeRegionsAsync(byte[][], boolean)} instead. */ + @Deprecated default Future mergeRegionsAsync(byte[] nameOfRegionA, byte[] nameOfRegionB, boolean forcible) throws IOException { byte[][] nameofRegionsToMerge = new byte[2][]; @@ -1423,11 +1426,7 @@ public interface Admin extends Abortable, Closeable { } /** - * Merge regions. Asynchronous operation. - *

- * You may get a {@code DoNotRetryIOException} if you pass more than two regions in but the master - * does not support merging more than two regions. At least till 2.2.0, we still only support - * merging two regions. + * Merge multiple regions (>=2). Asynchronous operation. * @param nameofRegionsToMerge encoded or full name of daughter regions * @param forcible true if do a compulsory merge, otherwise we will only merge * adjacent regions diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index c67dc2299a5c193f9a587a58d14828ed0f907f64..1aa76232293bc02dd3d14f7e567851c80fffad5f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -521,18 +521,17 @@ public interface AsyncAdmin { * @param nameOfRegionB encoded or full name of region b * @param forcible true if do a compulsory merge, otherwise we will only merge two adjacent * regions + * @deprecated since 2.3.0 and will be removed in 4.0.0.Use {@link #mergeRegions(List, boolean)} + * instead. */ + @Deprecated default CompletableFuture mergeRegions(byte[] nameOfRegionA, byte[] nameOfRegionB, boolean forcible) { return mergeRegions(Arrays.asList(nameOfRegionA, nameOfRegionB), forcible); } /** - * Merge regions. - *

- * You may get a {@code DoNotRetryIOException} if you pass more than two regions in but the master - * does not support merging more than two regions. At least till 2.2.0, we still only support - * merging two regions. + * Merge multiple regions (>=2). * @param nameOfRegionsToMerge encoded or full name of daughter regions * @param forcible true if do a compulsory merge, otherwise we will only merge two adjacent * regions diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index de6ccb4c723a978040875c4174732060398ee601..f60838e3f12a8d9a0fbbfae5fe6e491d21482bd3 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -514,11 +514,29 @@ module Hbase end #---------------------------------------------------------------------------------------------- - # Merge two regions - def merge_region(region_a_name, region_b_name, force) - @admin.mergeRegions(region_a_name.to_java_bytes, - region_b_name.to_java_bytes, - java.lang.Boolean.valueOf(force)) + # Merge multiple regions + def merge_region(regions, force) + unless regions.is_a?(Array) + raise(ArgumentError, "Type of #{regions.inspect} is #{regions.class}, but expected Array") + end + region_array = Java::byte[][regions.length].new + i = 0 + while i < regions.length + unless regions[i].is_a?(String) + raise( + ArgumentError, + "Type of #{regions[i].inspect} is #{regions[i].class}, but expected String" + ) + end + region_array[i] = regions[i].to_java_bytes + i += 1 + end + org.apache.hadoop.hbase.util.FutureUtils.get( + @admin.mergeRegionsAsync( + region_array, + java.lang.Boolean.valueOf(force) + ) + ) end #---------------------------------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands.rb b/hbase-shell/src/main/ruby/shell/commands.rb index 4fdc8b544ab0830418d2a9e768679f30b987b729..a40f737e79088b34ba81c83c73cb7f661885e2c7 100644 --- a/hbase-shell/src/main/ruby/shell/commands.rb +++ b/hbase-shell/src/main/ruby/shell/commands.rb @@ -137,7 +137,11 @@ module Shell raise "Table #{cause.message} should be disabled!" end if cause.is_a?(org.apache.hadoop.hbase.UnknownRegionException) - raise "Unknown region #{args.first}!" + raise cause.message + end + if cause.is_a?(org.apache.hadoop.hbase.exceptions.MergeRegionException) + strs = cause.message.split("\n") + raise(strs[0]).to_s unless strs.empty? end if cause.is_a?(org.apache.hadoop.hbase.NamespaceNotFoundException) s = /.*NamespaceNotFoundException: (?[^\n]+).*/.match(cause.message) diff --git a/hbase-shell/src/main/ruby/shell/commands/merge_region.rb b/hbase-shell/src/main/ruby/shell/commands/merge_region.rb index b4f6cae82050b5569aa08a07d186730ee072ae6b..ed17236277165f43c93420cfa73bce794fba7f44 100644 --- a/hbase-shell/src/main/ruby/shell/commands/merge_region.rb +++ b/hbase-shell/src/main/ruby/shell/commands/merge_region.rb @@ -22,7 +22,7 @@ module Shell class MergeRegion < Command def help <<-EOF -Merge two regions. Passing 'true' as the optional third parameter will force +Merge multiple (2 or more) regions. Passing 'true' as the optional third parameter will force a merge ('force' merges regardless else merge will fail unless passed adjacent regions. 'force' is for expert use only). @@ -31,18 +31,42 @@ region name is the hash suffix on region names: e.g. if the region name were TestTable,0094429456,1289497600452.527db22f95c8a9e0116f0cc13c680396. then the encoded region name portion is 527db22f95c8a9e0116f0cc13c680396 +You can either pass the list of regions as comma separated values or as an +array of regions as shown: + Examples: hbase> merge_region 'FULL_REGIONNAME', 'FULL_REGIONNAME' - hbase> merge_region 'FULL_REGIONNAME', 'FULL_REGIONNAME', true + hbase> merge_region 'FULL_REGIONNAME', 'FULL_REGIONNAME', 'FULL_REGIONNAME', ... + hbase> merge_region 'FULL_REGIONNAME', 'FULL_REGIONNAME', 'FULL_REGIONNAME', ..., true + + hbase> merge_region ['FULL_REGIONNAME', 'FULL_REGIONNAME'] + hbase> merge_region ['FULL_REGIONNAME', 'FULL_REGIONNAME', 'FULL_REGIONNAME', ...] + hbase> merge_region ['FULL_REGIONNAME', 'FULL_REGIONNAME', 'FULL_REGIONNAME', ...], true hbase> merge_region 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME' - hbase> merge_region 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', true + hbase> merge_region 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', ... + hbase> merge_region 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', ..., true + + hbase> merge_region ['ENCODED_REGIONNAME', 'ENCODED_REGIONNAME'] + hbase> merge_region ['ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', ...] + hbase> merge_region ['ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', 'ENCODED_REGIONNAME', ...], true EOF end - def command(region_a_name, region_b_name, force = 'false') - admin.merge_region(region_a_name, region_b_name, force) + def command(*args) + args = args.flatten.compact + args_len = args.length + raise(ArgumentError, 'Must pass at least 2 regions to merge') unless args_len > 1 + force = false + if(args_len > 2) + last = args[args_len-1] + if [true, false].include? last + force = last + args = args[0...-1] + end + end + admin.merge_region(args, force) end end end diff --git a/hbase-shell/src/test/ruby/hbase/admin_test.rb b/hbase-shell/src/test/ruby/hbase/admin_test.rb index c8f56a048372517fe0c4315ae1dffe03d1bec866..7d326453915184a34871f1107a3ec7c19b452e22 100644 --- a/hbase-shell/src/test/ruby/hbase/admin_test.rb +++ b/hbase-shell/src/test/ruby/hbase/admin_test.rb @@ -572,6 +572,61 @@ module Hbase define_test "list regions should allow table name" do command(:list_regions, @test_name) end + + define_test 'merge regions' do + @t_name = 'hbase_shell_merge' + @t_name2 = 'hbase_shell_merge_2' + drop_test_table(@t_name) + drop_test_table(@t_name2) + admin.create(@t_name, 'a', NUMREGIONS => 10, SPLITALGO => 'HexStringSplit') + r1 = command(:locate_region, @t_name, '1') + r2 = command(:locate_region, @t_name, '2') + r3 = command(:locate_region, @t_name, '4') + r4 = command(:locate_region, @t_name, '5') + r5 = command(:locate_region, @t_name, '7') + r6 = command(:locate_region, @t_name, '8') + region1 = r1.getRegion.getRegionNameAsString + region2 = r2.getRegion.getRegionNameAsString + region3 = r3.getRegion.getRegionNameAsString + region4 = r4.getRegion.getRegionNameAsString + region5 = r5.getRegion.getRegionNameAsString + region6 = r6.getRegion.getRegionNameAsString + # only 1 region + assert_raise(ArgumentError) do + command(:merge_region, 'a') + end + # only 1 region with force=true + assert_raise(ArgumentError) do + command(:merge_region, 'a', true) + end + # non-existing region + assert_raise(RuntimeError) do + command(:merge_region, 'a','b') + end + # duplicate regions + assert_raise(RuntimeError) do + command(:merge_region, region1,region1,region1) + end + # 3 non-adjacent regions without forcible=true + assert_raise(RuntimeError) do + command(:merge_region, region1,region2,region4) + end + # 2 adjacent regions + command(:merge_region, region1,region2) + # 3 non-adjacent regions with forcible=true + command(:merge_region, region3,region5,region6, true) + + admin.create(@t_name2, 'a', NUMREGIONS => 5, SPLITALGO => 'HexStringSplit') + r1 = command(:locate_region, @t_name2, '1') + r2 = command(:locate_region, @t_name2, '4') + r3 = command(:locate_region, @t_name2, '7') + region1 = r1.getRegion.getRegionNameAsString + region2 = r2.getRegion.getRegionNameAsString + region3 = r3.getRegion.getRegionNameAsString + + # accept array of regions + command(:merge_region, [region1,region2,region3]) + end end # Simple administration methods tests -- 2.20.1 (Apple Git-117)