From 6d0567a6fed3bcc1577a9e47a2903a6ab5d0c3fd Mon Sep 17 00:00:00 2001 From: TAK LON WU Date: Wed, 18 Jul 2018 18:00:48 -0700 Subject: [PATCH] HBASE-21011 Provide CLI option to run oldwals and hfiles cleaner separately when cleaner chore is disabled --- .../java/org/apache/hadoop/hbase/client/Admin.java | 6 +++- .../org/apache/hadoop/hbase/client/AsyncAdmin.java | 4 ++- .../hadoop/hbase/client/AsyncHBaseAdmin.java | 6 ++-- .../org/apache/hadoop/hbase/client/HBaseAdmin.java | 10 +++++-- .../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 5 ++-- .../hbase/shaded/protobuf/RequestConverter.java | 11 ++------ .../src/main/protobuf/HBase.proto | 7 +++++ .../src/main/protobuf/Master.proto | 1 + .../hadoop/hbase/master/MasterRpcServices.java | 33 +++++++++++++++++++++- .../hbase/security/access/TestRpcAccessChecks.java | 4 ++- hbase-shell/src/main/ruby/hbase/admin.rb | 6 ++-- .../main/ruby/shell/commands/cleaner_chore_run.rb | 12 ++++++-- 12 files changed, 83 insertions(+), 22 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index a43a0b2..9074db4 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -59,6 +59,8 @@ import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException; import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CleanerType; + /** * The administrative API for HBase. Obtain an instance from {@link Connection#getAdmin()} and * call {@link #close()} when done. @@ -1266,10 +1268,12 @@ public interface Admin extends Abortable, Closeable { /** * Ask for cleaner chore to run. * + * @param cleanerType HFILES, OLDWALS, or DEFAULT that first execute HFILES cleaner then OLDWALS + * cleaner chore * @return true if cleaner chore ran, false otherwise * @throws IOException */ - boolean runCleanerChore() throws IOException; + boolean runCleanerChore(final CleanerType cleanerType) throws IOException; /** * Query on the cleaner chore state (Enabled/Disabled?). diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index 739c78a..819ae94 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -46,6 +46,8 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; import org.apache.hadoop.hbase.replication.SyncReplicationState; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CleanerType; + /** * The asynchronous administrative API for HBase. * @since 2.0.0 @@ -1169,7 +1171,7 @@ public interface AsyncAdmin { * @return true if cleaner chore ran, false otherwise. The return value will be wrapped by a * {@link CompletableFuture} */ - CompletableFuture runCleanerChore(); + CompletableFuture runCleanerChore(CleanerType cleanerType); /** * Turn the catalog janitor on/off. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index 39eda07..7653d0d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -44,6 +44,8 @@ import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; import org.apache.hadoop.hbase.replication.SyncReplicationState; import org.apache.yetus.audience.InterfaceAudience; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CleanerType; + /** * Just a wrapper of {@link RawAsyncHBaseAdmin}. The difference is that users need to provide a * thread pool when constructing this class, and the callback methods registered to the returned @@ -708,8 +710,8 @@ class AsyncHBaseAdmin implements AsyncAdmin { } @Override - public CompletableFuture runCleanerChore() { - return wrap(rawAdmin.runCleanerChore()); + public CompletableFuture runCleanerChore(CleanerType cleanerType) { + return wrap(rawAdmin.runCleanerChore(cleanerType)); } @Override diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 856b118..9e1f036 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -30,6 +30,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.Callable; @@ -127,6 +128,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CleanerType; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ProcedureDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.TableSchema; @@ -1662,12 +1664,16 @@ public class HBaseAdmin implements Admin { }); } + public boolean runCleanerChore(final String cleanerType) throws IOException { + return runCleanerChore(CleanerType.valueOf(cleanerType.toUpperCase(Locale.ROOT))); + } + @Override - public boolean runCleanerChore() throws IOException { + public boolean runCleanerChore(final CleanerType cleanerType) throws IOException { return executeCallable(new MasterCallable(getConnection(), getRpcControllerFactory()) { @Override public Boolean rpcCall() throws Exception { return master.runCleanerChore(getRpcController(), - RequestConverter.buildRunCleanerChoreRequest()).getCleanerChoreRan(); + RequestConverter.buildRunCleanerChoreRequest(cleanerType)).getCleanerChoreRan(); } }); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index 4f73909..66a79f9 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -122,6 +122,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.StopServerR import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.StopServerResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.UpdateConfigurationRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.UpdateConfigurationResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CleanerType; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ProcedureDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.TableSchema; @@ -3174,13 +3175,13 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { } @Override - public CompletableFuture runCleanerChore() { + public CompletableFuture runCleanerChore(CleanerType cleanerType) { return this . newMasterCaller() .action( (controller, stub) -> this . call(controller, stub, - RequestConverter.buildRunCleanerChoreRequest(), + RequestConverter.buildRunCleanerChoreRequest(cleanerType), (s, c, req, done) -> s.runCleanerChore(c, req, done), (resp) -> resp.getCleanerChoreRan())).call(); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java index db07bab..db13f2d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java @@ -91,6 +91,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationPr import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.RegionAction; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CleanerType; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CompareType; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType; @@ -1560,17 +1561,11 @@ public final class RequestConverter { } /** - * @see #buildRunCleanerChoreRequest() - */ - private static final RunCleanerChoreRequest CLEANER_CHORE_REQUEST = - RunCleanerChoreRequest.newBuilder().build(); - - /** * Creates a request for running cleaner chore * @return A {@link RunCleanerChoreRequest} */ - public static RunCleanerChoreRequest buildRunCleanerChoreRequest() { - return CLEANER_CHORE_REQUEST; + public static RunCleanerChoreRequest buildRunCleanerChoreRequest(CleanerType cleanerType) { + return RunCleanerChoreRequest.newBuilder().setType(cleanerType).build(); } /** diff --git a/hbase-protocol-shaded/src/main/protobuf/HBase.proto b/hbase-protocol-shaded/src/main/protobuf/HBase.proto index d06bc8b..fe2b5c5 100644 --- a/hbase-protocol-shaded/src/main/protobuf/HBase.proto +++ b/hbase-protocol-shaded/src/main/protobuf/HBase.proto @@ -273,4 +273,11 @@ message RegionLocation { required RegionInfo region_info = 1; optional ServerName server_name = 2; required int64 seq_num = 3; +} + +/* cleaner chore type */ +enum CleanerType { + DEFAULT = 0; + HFILES = 1; + OLDWALS = 2; } \ No newline at end of file diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto index c2ab180..90be3dd 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Master.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto @@ -373,6 +373,7 @@ message IsCatalogJanitorEnabledResponse { } message RunCleanerChoreRequest { + optional CleanerType type = 1 [default = DEFAULT]; } message RunCleanerChoreResponse { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index a4d9ff8..4f6a7ac 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -115,6 +115,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CleanerType; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameStringPair; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ProcedureDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType; @@ -1432,7 +1433,37 @@ public class MasterRpcServices extends RSRpcServices public RunCleanerChoreResponse runCleanerChore(RpcController c, RunCleanerChoreRequest req) throws ServiceException { rpcPreCheck("runCleanerChore"); - boolean result = master.getHFileCleaner().runCleaner() && master.getLogCleaner().runCleaner(); + boolean result = false; + boolean isCleanerEnabled = + master.getHFileCleaner().getEnabled() && master.getLogCleaner().getEnabled(); + + if (isCleanerEnabled) { + LOG.warn("Cleaner(s) cannot be ran because cleaner chore has been enabled, please consider " + + "disable it."); + } else { + CleanerType cleanerType = req.getType(); + switch (cleanerType) { + case HFILES: + LOG.debug("Running HFiles cleaner"); + result = master.getHFileCleaner().runCleaner(); + break; + case OLDWALS: + LOG.debug("Running OldWals cleaner"); + result = master.getLogCleaner().runCleaner(); + break; + case DEFAULT: + LOG.debug("Running HFiles and OldWals cleaners"); + result = master.getHFileCleaner().runCleaner() && master.getLogCleaner().runCleaner(); + break; + default: + LOG.error("Unknown type `" + cleanerType + "` from run cleaner chore request."); + } + } + + if (!isCleanerEnabled && !result) { + LOG.warn("Cleaner(s) ran but failed to be completed"); + } + return ResponseConverter.buildRunCleanerChoreResponse(result); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestRpcAccessChecks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestRpcAccessChecks.java index 55873bb..daaf1a3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestRpcAccessChecks.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestRpcAccessChecks.java @@ -65,6 +65,8 @@ import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CleanerType; + /** * This class tests operations in MasterRpcServices which require ADMIN access. * It doesn't test all operations which require ADMIN access, only those which get vetted within @@ -223,7 +225,7 @@ public class TestRpcAccessChecks { @Test public void testRunCleanerChore() throws Exception { - verifyAdminCheckForAction((admin) -> admin.runCleanerChore()); + verifyAdminCheckForAction((admin) -> admin.runCleanerChore(CleanerType.DEFAULT)); } @Test diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 75d2de3..42298ce 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -272,8 +272,10 @@ module Hbase #---------------------------------------------------------------------------------------------- # Request cleaner chore to run (for garbage collection of HFiles and WAL files) - def cleaner_chore_run - @admin.runCleanerChore + def cleaner_chore_run(type = 'default') + raise(ArgumentError, 'Cleaner type name must be of type String') unless \ + type.is_a?(String) + @admin.runCleanerChore(type) end #---------------------------------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_run.rb b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_run.rb index 210f388..a484d1f 100644 --- a/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_run.rb +++ b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_run.rb @@ -24,12 +24,20 @@ module Shell Cleaner chore command for garbage collection of HFiles and WAL files. hbase> cleaner_chore_run + hbase> cleaner_chore_run 'hfiles' + hbase> cleaner_chore_run 'oldwals' EOF end - def command - admin.cleaner_chore_run + def command(type = 'default') + ret = admin.cleaner_chore_run(type) + if ret + puts 'Ran the cleaner(s) and succeeded.' + else + puts 'Couldn\'t run the cleaner(s) if cleaner chore is enabled or' \ + ' ran but failed.' + end end end end -- 2.10.1 (Apple Git-78)