From 30146cce7df5fd93e8ea1d686300f15dd2ca793f Mon Sep 17 00:00:00 2001 From: Toshihiro Suzuki Date: Wed, 18 Apr 2018 14:47:04 +0900 Subject: [PATCH] HBASE-20293 get_splits returns duplicate split points when region replication is on --- hbase-shell/src/main/ruby/hbase/table.rb | 25 ++++++++++++---------- .../src/main/ruby/shell/commands/get_splits.rb | 3 +-- hbase-shell/src/test/ruby/hbase/table_test.rb | 18 +++++++++++++++- hbase-shell/src/test/ruby/test_helper.rb | 11 ++++++++++ 4 files changed, 43 insertions(+), 14 deletions(-) diff --git a/hbase-shell/src/main/ruby/hbase/table.rb b/hbase-shell/src/main/ruby/hbase/table.rb index 3e3fb8e..aaaa877 100644 --- a/hbase-shell/src/main/ruby/hbase/table.rb +++ b/hbase-shell/src/main/ruby/hbase/table.rb @@ -20,6 +20,7 @@ include Java java_import org.apache.hadoop.hbase.util.Bytes +java_import org.apache.hadoop.hbase.client.RegionReplicaUtil # Wrapper for org.apache.hadoop.hbase.client.Table @@ -49,7 +50,7 @@ module Hbase self.class_eval do define_method method do |*args| @shell.internal_command(shell_command, internal_method_name, self, *args) - end + end end end @@ -143,7 +144,7 @@ EOF end #Case where attributes are specified without timestamp if timestamp.kind_of?(Hash) - timestamp.each do |k, v| + timestamp.each do |k, v| if k == 'ATTRIBUTES' set_attributes(p, v) elsif k == 'VISIBILITY' @@ -186,11 +187,11 @@ EOF end d = org.apache.hadoop.hbase.client.Delete.new(row.to_s.to_java_bytes, timestamp) if temptimestamp.kind_of?(Hash) - temptimestamp.each do |k, v| + temptimestamp.each do |k, v| if v.kind_of?(String) set_cell_visibility(d, v) if v end - end + end end if args.any? visibility = args[VISIBILITY] @@ -452,7 +453,8 @@ EOF raise ArgumentError.new("COLUMNS must be specified as a String or an Array") end - scan = if stoprow + scan = + if stoprow org.apache.hadoop.hbase.client.Scan.new(startrow.to_java_bytes, stoprow.to_java_bytes) else org.apache.hadoop.hbase.client.Scan.new(startrow.to_java_bytes) @@ -718,13 +720,14 @@ EOF #---------------------------------------------------------------------------------------------- # Get the split points for the table - def _get_splits_internal() - locator = @table.getRegionLocator() - locator.getAllRegionLocations() - .map { |i| Bytes.toStringBinary(i.getRegionInfo().getStartKey) } - .delete_if { |k| k == "" } + def _get_splits_internal + l = @table.getRegionLocator + l.getAllRegionLocations. + select { |s| RegionReplicaUtil.isDefaultReplica(s.getRegionInfo) }. + map { |i| Bytes.toStringBinary(i.getRegionInfo.getStartKey) }. + delete_if { |k| k == "" } ensure - locator.close() + l.close end end # rubocop:enable Metrics/ClassLength diff --git a/hbase-shell/src/main/ruby/shell/commands/get_splits.rb b/hbase-shell/src/main/ruby/shell/commands/get_splits.rb index 26be15f..2a74b0f 100644 --- a/hbase-shell/src/main/ruby/shell/commands/get_splits.rb +++ b/hbase-shell/src/main/ruby/shell/commands/get_splits.rb @@ -38,8 +38,7 @@ EOF def get_splits(table) splits = table._get_splits_internal() - puts(format('Total number of splits = %d', - numsplits: (splits.size + 1))) + puts(format('Total number of splits = %d', splits.size + 1)) splits end end diff --git a/hbase-shell/src/test/ruby/hbase/table_test.rb b/hbase-shell/src/test/ruby/hbase/table_test.rb index 6ffdf89..a631fc5 100644 --- a/hbase-shell/src/test/ruby/hbase/table_test.rb +++ b/hbase-shell/src/test/ruby/hbase/table_test.rb @@ -188,6 +188,7 @@ module Hbase end # Complex data management methods tests + # rubocop:disable Metrics/ClassLength class TableComplexMethodsTest < Test::Unit::TestCase include TestHelpers @@ -302,7 +303,8 @@ module Hbase assert_not_nil(res['x:b']) end - define_test "get should work with hash columns spec and TIMESTAMP and AUTHORIZATIONS" do + define_test 'get should work with hash columns spec and TIMESTAMP and' \ + ' AUTHORIZATIONS' do res = @test_table._get_internal('1', TIMESTAMP => 1234, AUTHORIZATIONS=>['PRIVATE']) assert_nil(res) end @@ -635,5 +637,19 @@ module Hbase assert_equal(0, splits.size) assert_equal([], splits) end + + define_test 'Split count for a table with region replicas' do + @test_table_name = 'tableWithRegionReplicas' + create_test_table_with_region_replicas(@test_table_name, 3, + SPLITS => ['10']) + @table = table(@test_table_name) + splits = @table._get_splits_internal + # In this case, total splits should be 1 even if the number of region + # replicas is 3. + assert_equal(1, splits.size) + assert_equal(['10'], splits) + drop_test_table(@test_table_name) + end end + # rubocop:enable Metrics/ClassLength end diff --git a/hbase-shell/src/test/ruby/test_helper.rb b/hbase-shell/src/test/ruby/test_helper.rb index b4bec90..c947439 100644 --- a/hbase-shell/src/test/ruby/test_helper.rb +++ b/hbase-shell/src/test/ruby/test_helper.rb @@ -107,6 +107,17 @@ module Hbase end end + def create_test_table_with_region_replicas(name, num_of_replicas, splits) + # Create the table if needed + unless admin.exists?(name) + admin.create name, 'f1', { REGION_REPLICATION => num_of_replicas }, + splits + end + + # Enable the table if needed + admin.enable(name) unless admin.enabled?(name) + end + def drop_test_table(name) return unless admin.exists?(name) begin -- 2.10.1 (Apple Git-78)