From e0def8aa596ac36701d606e997148704d7659e63 Mon Sep 17 00:00:00 2001 From: stack Date: Tue, 21 Oct 2014 09:09:43 -0700 Subject: [PATCH] Add doc to the refguide, change command names to drop the _rs_ --- hbase-shell/src/main/ruby/hbase/admin.rb | 81 +++++++++++++--------- hbase-shell/src/main/ruby/shell.rb | 9 +++ .../main/ruby/shell/commands/update_all_config.rb | 39 +++++++++++ .../src/main/ruby/shell/commands/update_config.rb | 40 +++++++++++ src/main/docbkx/configuration.xml | 18 +++++ 5 files changed, 153 insertions(+), 34 deletions(-) create mode 100644 hbase-shell/src/main/ruby/shell/commands/update_all_config.rb create mode 100644 hbase-shell/src/main/ruby/shell/commands/update_config.rb diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index a242846..2a5876a 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -22,6 +22,7 @@ java_import java.util.Arrays java_import org.apache.hadoop.hbase.util.Pair java_import org.apache.hadoop.hbase.util.RegionSplitter java_import org.apache.hadoop.hbase.util.Bytes +java_import org.apache.hadoop.hbase.ServerName java_import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos::SnapshotDescription # Wrapper for org.apache.hadoop.hbase.client.HBaseAdmin @@ -211,7 +212,7 @@ module Hbase unless arg.kind_of?(String) || arg.kind_of?(Hash) raise(ArgumentError, "#{arg.class} of #{arg.inspect} is not of Hash or String type") end - + # First, handle all the cases where arg is a column family. if arg.kind_of?(String) or arg.has_key?(NAME) # If the arg is a string, default action is to add a column to the table. @@ -223,14 +224,14 @@ module Hbase if arg.has_key?(REGION_REPLICATION) region_replication = JInteger.valueOf(arg.delete(REGION_REPLICATION)) htd.setRegionReplication(region_replication) - end - + end + # Get rid of the "METHOD", which is deprecated for create. # We'll do whatever it used to do below if it's table_att. if (method = arg.delete(METHOD)) raise(ArgumentError, "table_att is currently the only supported method") unless method == 'table_att' end - + # The hash is not a column family. Figure out what's in it. # First, handle splits. if arg.has_key?(SPLITS_FILE) @@ -245,7 +246,7 @@ module Hbase htd.setValue(SPLITS_FILE, arg[SPLITS_FILE]) end - if arg.has_key?(SPLITS) + if arg.has_key?(SPLITS) splits = Java::byte[][arg[SPLITS].size].new idx = 0 arg.delete(SPLITS).each do |split| @@ -261,9 +262,9 @@ module Hbase split_algo = RegionSplitter.newSplitAlgoInstance(@conf, arg.delete(SPLITALGO)) splits = split_algo.split(JInteger.valueOf(num_regions)) end - + # Done with splits; apply formerly-table_att parameters. - htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER] + htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER] htd.setMaxFileSize(JLong.valueOf(arg.delete(MAX_FILESIZE))) if arg[MAX_FILESIZE] htd.setReadOnly(JBoolean.valueOf(arg.delete(READONLY))) if arg[READONLY] htd.setCompactionEnabled(JBoolean.valueOf(arg[COMPACTION_ENABLED])) if arg[COMPACTION_ENABLED] @@ -272,15 +273,15 @@ module Hbase htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf(arg.delete(DURABILITY))) if arg[DURABILITY] set_user_metadata(htd, arg.delete(METADATA)) if arg[METADATA] set_descriptor_config(htd, arg.delete(CONFIGURATION)) if arg[CONFIGURATION] - + arg.each_key do |ignored_key| puts("An argument ignored (unknown or overridden): %s" % [ ignored_key ]) end end - + # Fail if no column families defined raise(ArgumentError, "Table must have at least one column family") if !has_columns - + if splits.nil? # Perform the create table call @admin.createTable(htd) @@ -289,15 +290,15 @@ module Hbase @admin.createTable(htd, splits) end end - + #---------------------------------------------------------------------------------------------- # Closes a region. # If server name is nil, we presume region_name is full region name (HRegionInfo.getRegionName). # If server name is not nil, we presume it is the region's encoded name (HRegionInfo.getEncodedName) def close_region(region_name, server) - if (server == nil || !closeEncodedRegion?(region_name, server)) + if (server == nil || !closeEncodedRegion?(region_name, server)) @admin.closeRegion(region_name, server) - end + end end #---------------------------------------------------------------------------------------------- @@ -318,7 +319,7 @@ module Hbase def move(encoded_region_name, server = nil) @admin.move(encoded_region_name.to_java_bytes, server ? server.to_java_bytes: nil) end - + #---------------------------------------------------------------------------------------------- # Merge two regions def merge_region(encoded_region_a_name, encoded_region_b_name, force) @@ -433,14 +434,14 @@ module Hbase # Process all args args.each do |arg| - - + + # Normalize args to support column name only alter specs arg = { NAME => arg } if arg.kind_of?(String) # Normalize args to support shortcut delete syntax arg = { METHOD => 'delete', NAME => arg['delete'] } if arg['delete'] - + # There are 3 possible options. # 1) Column family spec. Distinguished by having a NAME and no METHOD. method = arg.delete(METHOD) @@ -459,12 +460,12 @@ module Hbase puts "Updating all regions with the new schema..." alter_status(table_name) end - + # We bypass descriptor when adding column families; refresh it to apply other args correctly. htd = @admin.getTableDescriptor(table_name.to_java_bytes) next end - + # 2) Method other than table_att, with some args. name = arg.delete(NAME) if method != nil and method != "table_att" @@ -484,26 +485,26 @@ module Hbase else raise ArgumentError, "Unknown method: #{method}" end - + arg.each_key do |unknown_key| puts("Unknown argument ignored: %s" % [unknown_key]) end - + if wait == true puts "Updating all regions with the new schema..." alter_status(table_name) end - + if method == "delete" # We bypass descriptor when deleting column families; refresh it to apply other args correctly. htd = @admin.getTableDescriptor(table_name.to_java_bytes) end - next + next end - + # 3) Some args for the table, optionally with METHOD => table_att (deprecated) raise(ArgumentError, "NAME argument in an unexpected place") if name - htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER] + htd.setOwnerString(arg.delete(OWNER)) if arg[OWNER] htd.setMaxFileSize(JLong.valueOf(arg.delete(MAX_FILESIZE))) if arg[MAX_FILESIZE] htd.setReadOnly(JBoolean.valueOf(arg.delete(READONLY))) if arg[READONLY] htd.setCompactionEnabled(JBoolean.valueOf(arg[COMPACTION_ENABLED])) if arg[COMPACTION_ENABLED] @@ -544,17 +545,17 @@ module Hbase valid_coproc_keys << key end end - + valid_coproc_keys.each do |key| arg.delete(key) end @admin.modifyTable(table_name.to_java_bytes, htd) - + arg.each_key do |unknown_key| puts("Unknown argument ignored: %s" % [unknown_key]) end - + if wait == true puts "Updating all regions with the new schema..." alter_status(table_name) @@ -621,7 +622,7 @@ module Hbase def exists?(table_name) @admin.tableExists(table_name) end - + #---------------------------------------------------------------------------------------------- # Is table enabled def enabled?(table_name) @@ -632,7 +633,7 @@ module Hbase #Is supplied region name is encoded region name def closeEncodedRegion?(region_name, server) @admin.closeRegionWithEncodedRegionName(region_name, server) - end + end #---------------------------------------------------------------------------------------------- # Return a new HColumnDescriptor made of passed args @@ -661,8 +662,8 @@ module Hbase if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER) bloomtype = arg.delete(org.apache.hadoop.hbase.HColumnDescriptor::BLOOMFILTER).upcase unless org.apache.hadoop.hbase.regionserver.BloomType.constants.include?(bloomtype) - raise(ArgumentError, "BloomFilter type #{bloomtype} is not supported. Use one of " + org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(" ")) - else + raise(ArgumentError, "BloomFilter type #{bloomtype} is not supported. Use one of " + org.apache.hadoop.hbase.regionserver.StoreFile::BloomType.constants.join(" ")) + else family.setBloomFilterType(org.apache.hadoop.hbase.regionserver.BloomType.valueOf(bloomtype)) end end @@ -691,7 +692,7 @@ module Hbase arg.each_key do |unknown_key| puts("Unknown argument ignored for column family %s: %s" % [name, unknown_key]) end - + return family end @@ -726,7 +727,7 @@ module Hbase descriptor.setValue(k, v) end end - + #---------------------------------------------------------------------------------------------- # Take a snapshot of specified table def snapshot(table, snapshot_name, *args) @@ -783,6 +784,18 @@ module Hbase end #---------------------------------------------------------------------------------------------- + # Updates the configuration of one regionserver. + def update_config(serverName) + @admin.updateConfiguration(ServerName.valueOf(serverName)); + end + + #---------------------------------------------------------------------------------------------- + # Updates the configuration of all the regionservers. + def update_all_config() + @admin.updateConfiguration(); + end + + #---------------------------------------------------------------------------------------------- # Returns namespace's structure description def describe_namespace(namespace_name) namespace = @admin.getNamespaceDescriptor(namespace_name) diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index 1cfd3d6..1c10bc1 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -361,6 +361,15 @@ Shell.load_command_group( ) Shell.load_command_group( + 'configuration', + :full_name => 'ONLINE CONFIGURATION TOOLS', + :commands => %w[ + update_config + update_all_config + ] +) + +Shell.load_command_group( 'quotas', :full_name => 'CLUSTER QUOTAS TOOLS', :commands => %w[ diff --git a/hbase-shell/src/main/ruby/shell/commands/update_all_config.rb b/hbase-shell/src/main/ruby/shell/commands/update_all_config.rb new file mode 100644 index 0000000..05295b7 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/update_all_config.rb @@ -0,0 +1,39 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class UpdateAllConfig < Command + def help + return <<-EOF +Reload a subset of configuration on all servers in the cluster. See +http://hbase.apache.org/book.html?dyn_config for more details. Here is how +you would run the command in the hbase shell: + hbase> update_all_config +EOF + end + + def command() + format_simple_command do + admin.update_all_config() + end + end + end + end +end diff --git a/hbase-shell/src/main/ruby/shell/commands/update_config.rb b/hbase-shell/src/main/ruby/shell/commands/update_config.rb new file mode 100644 index 0000000..9f65fdd --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/update_config.rb @@ -0,0 +1,40 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class UpdateConfig < Command + def help + return <<-EOF +Reload a subset of configuration on server 'servername' where servername is +host, port plus startcode. For example: host187.example.com,60020,1289493121758 +See http://hbase.apache.org/book.html?dyn_config for more details. Here is how +you would run the command in the hbase shell: + hbase> update_config 'servername' +EOF + end + + def command(serverName) + format_simple_command do + admin.update_config(serverName) + end + end + end + end +end diff --git a/src/main/docbkx/configuration.xml b/src/main/docbkx/configuration.xml index d3959b6..6907bb7 100644 --- a/src/main/docbkx/configuration.xml +++ b/src/main/docbkx/configuration.xml @@ -1633,5 +1633,23 @@ jconsole -J-Djavax.net.ssl.trustStore=/home/tianq/jconsoleKeyStore +
+ Dynamic Configuration + Changing Configuration Without Restarting Servers + Since HBase 1.0.0, it is possible to change a subset of the configuration without + requiring a server restart. In the hbase shell, there are new operators, + update_config and update_all_config that + will prompt a server or all servers to reload configuration. + Only a subset of all configurations can currently be changed in the running server. + Here is an incomplete list: + hbase.regionserver.thread.compaction.large, + hbase.regionserver.thread.compaction.small, + hbase.regionserver.thread.split, + hbase.regionserver.thread.merge, as well as compaction + policy and configurations and adjustment to offpeak hours. + For the full list consult the patch attached to + HBASE-12147 Porting Online Config Change from 89-fb. + +
-- 1.8.5.2 (Apple Git-48)