From f6034527e29b21b12f045e7012f205f585a8a274 Mon Sep 17 00:00:00 2001 From: TAK LON WU Date: Wed, 18 Jul 2018 18:00:48 -0700 Subject: [PATCH] HBASE-21011 Provide CLI option to run oldwals and hfiles cleaner separately when cleaner chore is disabled --- .../java/org/apache/hadoop/hbase/client/Admin.java | 5 +++- .../org/apache/hadoop/hbase/client/AsyncAdmin.java | 3 +- .../hadoop/hbase/client/AsyncHBaseAdmin.java | 5 ++-- .../org/apache/hadoop/hbase/client/HBaseAdmin.java | 10 +++++-- .../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 5 ++-- .../hbase/shaded/protobuf/RequestConverter.java | 11 ++------ .../src/main/protobuf/HBase.proto | 7 +++++ .../src/main/protobuf/Master.proto | 1 + .../hadoop/hbase/master/MasterRpcServices.java | 33 +++++++++++++++++++++- .../hbase/security/access/TestRpcAccessChecks.java | 3 +- hbase-shell/src/main/ruby/hbase/admin.rb | 5 ++-- .../main/ruby/shell/commands/cleaner_chore_run.rb | 12 ++++++-- 12 files changed, 77 insertions(+), 23 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java index a43a0b2..68404f1 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java @@ -52,6 +52,7 @@ import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; import org.apache.hadoop.hbase.replication.SyncReplicationState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CleanerType; import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException; import org.apache.hadoop.hbase.snapshot.SnapshotCreationException; @@ -1266,10 +1267,12 @@ public interface Admin extends Abortable, Closeable { /** * Ask for cleaner chore to run. * + * @param cleanerType HFILES, OLDWALS, or DEFAULT that first execute HFILES cleaner then OLDWALS + * cleaner chore * @return true if cleaner chore ran, false otherwise * @throws IOException */ - boolean runCleanerChore() throws IOException; + boolean runCleanerChore(final CleanerType cleanerType) throws IOException; /** * Query on the cleaner chore state (Enabled/Disabled?). diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java index 739c78a..0fc54bb 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java @@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.quotas.QuotaSettings; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; import org.apache.hadoop.hbase.replication.SyncReplicationState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CleanerType; import org.apache.yetus.audience.InterfaceAudience; /** @@ -1169,7 +1170,7 @@ public interface AsyncAdmin { * @return true if cleaner chore ran, false otherwise. The return value will be wrapped by a * {@link CompletableFuture} */ - CompletableFuture runCleanerChore(); + CompletableFuture runCleanerChore(CleanerType cleanerType); /** * Turn the catalog janitor on/off. diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java index 39eda07..6c1cd27 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.quotas.QuotaSettings; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.apache.hadoop.hbase.replication.ReplicationPeerDescription; import org.apache.hadoop.hbase.replication.SyncReplicationState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CleanerType; import org.apache.yetus.audience.InterfaceAudience; /** @@ -708,8 +709,8 @@ class AsyncHBaseAdmin implements AsyncAdmin { } @Override - public CompletableFuture runCleanerChore() { - return wrap(rawAdmin.runCleanerChore()); + public CompletableFuture runCleanerChore(CleanerType cleanerType) { + return wrap(rawAdmin.runCleanerChore(cleanerType)); } @Override diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java index 856b118..9e1f036 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java @@ -30,6 +30,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.Callable; @@ -127,6 +128,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CleanerType; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ProcedureDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.TableSchema; @@ -1662,12 +1664,16 @@ public class HBaseAdmin implements Admin { }); } + public boolean runCleanerChore(final String cleanerType) throws IOException { + return runCleanerChore(CleanerType.valueOf(cleanerType.toUpperCase(Locale.ROOT))); + } + @Override - public boolean runCleanerChore() throws IOException { + public boolean runCleanerChore(final CleanerType cleanerType) throws IOException { return executeCallable(new MasterCallable(getConnection(), getRpcControllerFactory()) { @Override public Boolean rpcCall() throws Exception { return master.runCleanerChore(getRpcController(), - RequestConverter.buildRunCleanerChoreRequest()).getCleanerChoreRan(); + RequestConverter.buildRunCleanerChoreRequest(cleanerType)).getCleanerChoreRan(); } }); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java index 4f73909..66a79f9 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java @@ -122,6 +122,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.StopServerR import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.StopServerResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.UpdateConfigurationRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.UpdateConfigurationResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CleanerType; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ProcedureDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.TableSchema; @@ -3174,13 +3175,13 @@ class RawAsyncHBaseAdmin implements AsyncAdmin { } @Override - public CompletableFuture runCleanerChore() { + public CompletableFuture runCleanerChore(CleanerType cleanerType) { return this . newMasterCaller() .action( (controller, stub) -> this . call(controller, stub, - RequestConverter.buildRunCleanerChoreRequest(), + RequestConverter.buildRunCleanerChoreRequest(cleanerType), (s, c, req, done) -> s.runCleanerChore(c, req, done), (resp) -> resp.getCleanerChoreRan())).call(); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java index db07bab..db13f2d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java @@ -91,6 +91,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationPr import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.RegionAction; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CleanerType; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CompareType; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType; @@ -1560,17 +1561,11 @@ public final class RequestConverter { } /** - * @see #buildRunCleanerChoreRequest() - */ - private static final RunCleanerChoreRequest CLEANER_CHORE_REQUEST = - RunCleanerChoreRequest.newBuilder().build(); - - /** * Creates a request for running cleaner chore * @return A {@link RunCleanerChoreRequest} */ - public static RunCleanerChoreRequest buildRunCleanerChoreRequest() { - return CLEANER_CHORE_REQUEST; + public static RunCleanerChoreRequest buildRunCleanerChoreRequest(CleanerType cleanerType) { + return RunCleanerChoreRequest.newBuilder().setType(cleanerType).build(); } /** diff --git a/hbase-protocol-shaded/src/main/protobuf/HBase.proto b/hbase-protocol-shaded/src/main/protobuf/HBase.proto index d06bc8b..fe2b5c5 100644 --- a/hbase-protocol-shaded/src/main/protobuf/HBase.proto +++ b/hbase-protocol-shaded/src/main/protobuf/HBase.proto @@ -273,4 +273,11 @@ message RegionLocation { required RegionInfo region_info = 1; optional ServerName server_name = 2; required int64 seq_num = 3; +} + +/* cleaner chore type */ +enum CleanerType { + DEFAULT = 0; + HFILES = 1; + OLDWALS = 2; } \ No newline at end of file diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto index c2ab180..624cb74 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Master.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto @@ -373,6 +373,7 @@ message IsCatalogJanitorEnabledResponse { } message RunCleanerChoreRequest { + required CleanerType type = 1; } message RunCleanerChoreResponse { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index a4d9ff8..35994fc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -115,6 +115,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos.RegionStoreSequenceIds; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CleanerType; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameStringPair; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ProcedureDescription; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType; @@ -1432,7 +1433,37 @@ public class MasterRpcServices extends RSRpcServices public RunCleanerChoreResponse runCleanerChore(RpcController c, RunCleanerChoreRequest req) throws ServiceException { rpcPreCheck("runCleanerChore"); - boolean result = master.getHFileCleaner().runCleaner() && master.getLogCleaner().runCleaner(); + boolean result = false; + boolean isCleanerEnabled = + master.getHFileCleaner().getEnabled() && master.getLogCleaner().getEnabled(); + + if (isCleanerEnabled) { + LOG.warn("Cleaner(s) cannot be ran because cleaner chore has been enabled, please consider " + + "disable it."); + } else { + CleanerType cleanerType = req.getType(); + switch (cleanerType) { + case HFILES: + LOG.debug("Running HFiles cleaner"); + result = master.getHFileCleaner().runCleaner(); + break; + case OLDWALS: + LOG.debug("Running OldWals cleaner"); + result = master.getLogCleaner().runCleaner(); + break; + case DEFAULT: + LOG.debug("Running HFiles and OldWals cleaners"); + result = master.getHFileCleaner().runCleaner() && master.getLogCleaner().runCleaner(); + break; + default: + LOG.error("Unknown type `" + cleanerType + "` from run cleaner chore request."); + } + } + + if (!isCleanerEnabled && !result) { + LOG.warn("Cleaner(s) ran but failed to be completed"); + } + return ResponseConverter.buildRunCleanerChoreResponse(result); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestRpcAccessChecks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestRpcAccessChecks.java index 55873bb..781c193 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestRpcAccessChecks.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestRpcAccessChecks.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos; import org.apache.hadoop.hbase.ipc.protobuf.generated.TestRpcServiceProtos; import org.apache.hadoop.hbase.security.AccessDeniedException; import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.CleanerType; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.SecurityTests; import org.apache.hadoop.hbase.util.Bytes; @@ -223,7 +224,7 @@ public class TestRpcAccessChecks { @Test public void testRunCleanerChore() throws Exception { - verifyAdminCheckForAction((admin) -> admin.runCleanerChore()); + verifyAdminCheckForAction((admin) -> admin.runCleanerChore(CleanerType.DEFAULT)); } @Test diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 75d2de3..2cf207b 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -272,8 +272,9 @@ module Hbase #---------------------------------------------------------------------------------------------- # Request cleaner chore to run (for garbage collection of HFiles and WAL files) - def cleaner_chore_run - @admin.runCleanerChore + def cleaner_chore_run(type = "default") + raise(ArgumentError, "Cleaner type name must be of type String") unless type.kind_of?(String) + @admin.runCleanerChore(type) end #---------------------------------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_run.rb b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_run.rb index 210f388..e3ff5d0 100644 --- a/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_run.rb +++ b/hbase-shell/src/main/ruby/shell/commands/cleaner_chore_run.rb @@ -24,12 +24,18 @@ module Shell Cleaner chore command for garbage collection of HFiles and WAL files. hbase> cleaner_chore_run - + hbase> cleaner_chore_run 'hfiles' + hbase> cleaner_chore_run 'oldwals' EOF end - def command - admin.cleaner_chore_run + def command(type = "default") + ret = admin.cleaner_chore_run(type) + if ret + puts 'Ran the cleaner(s) and succeeded.' + else + puts "Couldn't run the cleaner(s) if cleaner chore is enabled or ran but failed." + end end end end -- 2.10.1 (Apple Git-78)