From e931463950586a18e5f1b09c6e3ed71a55ef1612 Mon Sep 17 00:00:00 2001 From: Josh Elser Date: Mon, 1 May 2017 19:44:47 -0400 Subject: [PATCH] HBASE-17981 Consolidate the space quota shell commands --- .../hadoop/hbase/client/QuotaStatusCalls.java | 29 ------- .../apache/hadoop/hbase/quotas/QuotaTableUtil.java | 44 +++++------ .../shaded/protobuf/generated/AdminProtos.java | 90 +--------------------- .../src/main/protobuf/Admin.proto | 4 - .../src/main/protobuf/Quota.proto | 12 --- .../hadoop/hbase/regionserver/RSRpcServices.java | 30 -------- .../hadoop/hbase/master/MockRegionServer.java | 9 --- .../hadoop/hbase/quotas/TestQuotaStatusRPCs.java | 13 ++-- hbase-shell/src/main/ruby/hbase/quotas.rb | 18 +++-- hbase-shell/src/main/ruby/hbase_constants.rb | 1 + hbase-shell/src/main/ruby/shell.rb | 1 - .../ruby/shell/commands/list_quota_snapshots.rb | 48 +++++++++--- .../ruby/shell/commands/list_quota_table_sizes.rb | 8 +- .../ruby/shell/commands/list_quota_violations.rb | 48 ------------ .../src/test/ruby/hbase/quotas_test_no_cluster.rb | 19 ++++- 15 files changed, 101 insertions(+), 273 deletions(-) delete mode 100644 hbase-shell/src/main/ruby/shell/commands/list_quota_violations.rb diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java index af36d1ebad..70f6fb6088 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java @@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse; -import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse; @@ -127,32 +126,4 @@ public class QuotaStatusCalls { }; return ProtobufUtil.call(callable); } - - /** - * See {@link #getRegionServerSpaceQuotaEnforcements(ClusterConnection, RpcControllerFactory, int, ServerName)} - */ - public static GetSpaceQuotaEnforcementsResponse getRegionServerSpaceQuotaEnforcements( - ClusterConnection clusterConn, int timeout, ServerName sn) throws IOException { - RpcControllerFactory rpcController = clusterConn.getRpcControllerFactory(); - return getRegionServerSpaceQuotaEnforcements(clusterConn, rpcController, timeout, sn); - } - - /** - * Executes an RPC to the RegionServer identified by the {@code ServerName} to fetch its view on - * enforced space quotas. - */ - public static GetSpaceQuotaEnforcementsResponse getRegionServerSpaceQuotaEnforcements( - ClusterConnection conn, RpcControllerFactory factory, - int timeout, ServerName sn) throws IOException { - final AdminService.BlockingInterface admin = conn.getAdmin(sn); - Callable callable = - new Callable() { - @Override - public GetSpaceQuotaEnforcementsResponse call() throws Exception { - return admin.getSpaceQuotaEnforcements( - factory.newController(), RequestConverter.buildGetSpaceQuotaEnforcementsRequest()); - } - }; - return ProtobufUtil.call(callable); - } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java index ec480c474e..1b670e67a3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.QuotaStatusCalls; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.filter.CompareFilter; @@ -50,7 +51,6 @@ import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.filter.QualifierFilter; import org.apache.hadoop.hbase.filter.RegexStringComparator; import org.apache.hadoop.hbase.filter.RowFilter; -import org.apache.hadoop.hbase.ipc.RpcControllerFactory; import org.apache.hadoop.hbase.protobuf.ProtobufMagic; import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString; import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException; @@ -59,8 +59,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse; -import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse; -import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse.TableViolationPolicy; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse.TableQuotaSnapshot; @@ -240,6 +238,23 @@ public class QuotaTableUtil { } /** + * Fetches all {@link SpaceQuotaSnapshot} objects from the {@code hbase:quota} table. + * + * @param conn The HBase connection + * @return A map of table names and their computed snapshot. + */ + public static Map getSnapshots(Connection conn) throws IOException { + Map snapshots = new HashMap<>(); + try (Table quotaTable = conn.getTable(QUOTA_TABLE_NAME); + ResultScanner rs = quotaTable.getScanner(makeQuotaSnapshotScan())) { + for (Result r : rs) { + extractQuotaSnapshot(r, snapshots); + } + } + return snapshots; + } + + /** * Extracts the {@link SpaceViolationPolicy} and {@link TableName} from the provided * {@link Result} and adds them to the given {@link Map}. If the result does not contain * the expected information or the serialized policy in the value is invalid, this method @@ -453,29 +468,6 @@ public class QuotaTableUtil { } /** - * Fetches the active {@link SpaceViolationPolicy}'s that are being enforced on the - * given RegionServer. - */ - public static Map getRegionServerQuotaViolations( - Connection conn, ServerName regionServer) throws IOException { - if (!(conn instanceof ClusterConnection)) { - throw new IllegalArgumentException("Expected a ClusterConnection"); - } - ClusterConnection clusterConn = (ClusterConnection) conn; - RpcControllerFactory rpcController = clusterConn.getRpcControllerFactory(); - GetSpaceQuotaEnforcementsResponse response = - QuotaStatusCalls.getRegionServerSpaceQuotaEnforcements( - clusterConn, rpcController, 0, regionServer); - Map policies = new HashMap<>(); - for (TableViolationPolicy policy : response.getViolationPoliciesList()) { - policies.put( - ProtobufUtil.toTableName(policy.getTableName()), - ProtobufUtil.toViolationPolicy(policy.getViolationPolicy())); - } - return policies; - } - - /** * Returns the Master's view of a quota on the given {@code tableName} or null if the * Master has no quota information on that table. */ diff --git a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/AdminProtos.java b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/AdminProtos.java index 9145a63a0b..76ecb3d19b 100644 --- a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/AdminProtos.java +++ b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/AdminProtos.java @@ -24914,18 +24914,6 @@ public final class AdminProtos { org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest request, org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback done); - /** - *
-       ** Fetches the RegionServer's space quota active enforcements
-       * 
- * - * rpc GetSpaceQuotaEnforcements(.hbase.pb.GetSpaceQuotaEnforcementsRequest) returns (.hbase.pb.GetSpaceQuotaEnforcementsResponse); - */ - public abstract void getSpaceQuotaEnforcements( - org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller, - org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest request, - org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback done); - } public static org.apache.hadoop.hbase.shaded.com.google.protobuf.Service newReflectiveService( @@ -25083,14 +25071,6 @@ public final class AdminProtos { impl.getSpaceQuotaSnapshots(controller, request, done); } - @java.lang.Override - public void getSpaceQuotaEnforcements( - org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller, - org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest request, - org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback done) { - impl.getSpaceQuotaEnforcements(controller, request, done); - } - }; } @@ -25151,8 +25131,6 @@ public final class AdminProtos { return impl.getRegionLoad(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest)request); case 18: return impl.getSpaceQuotaSnapshots(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest)request); - case 19: - return impl.getSpaceQuotaEnforcements(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest)request); default: throw new java.lang.AssertionError("Can't get here."); } @@ -25205,8 +25183,6 @@ public final class AdminProtos { return org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest.getDefaultInstance(); case 18: return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest.getDefaultInstance(); - case 19: - return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest.getDefaultInstance(); default: throw new java.lang.AssertionError("Can't get here."); } @@ -25259,8 +25235,6 @@ public final class AdminProtos { return org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadResponse.getDefaultInstance(); case 18: return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse.getDefaultInstance(); - case 19: - return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse.getDefaultInstance(); default: throw new java.lang.AssertionError("Can't get here."); } @@ -25425,18 +25399,6 @@ public final class AdminProtos { org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest request, org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback done); - /** - *
-     ** Fetches the RegionServer's space quota active enforcements
-     * 
- * - * rpc GetSpaceQuotaEnforcements(.hbase.pb.GetSpaceQuotaEnforcementsRequest) returns (.hbase.pb.GetSpaceQuotaEnforcementsResponse); - */ - public abstract void getSpaceQuotaEnforcements( - org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller, - org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest request, - org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback done); - public static final org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.ServiceDescriptor getDescriptor() { @@ -25554,11 +25516,6 @@ public final class AdminProtos { org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcUtil.specializeCallback( done)); return; - case 19: - this.getSpaceQuotaEnforcements(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest)request, - org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcUtil.specializeCallback( - done)); - return; default: throw new java.lang.AssertionError("Can't get here."); } @@ -25611,8 +25568,6 @@ public final class AdminProtos { return org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadRequest.getDefaultInstance(); case 18: return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest.getDefaultInstance(); - case 19: - return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest.getDefaultInstance(); default: throw new java.lang.AssertionError("Can't get here."); } @@ -25665,8 +25620,6 @@ public final class AdminProtos { return org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.GetRegionLoadResponse.getDefaultInstance(); case 18: return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse.getDefaultInstance(); - case 19: - return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse.getDefaultInstance(); default: throw new java.lang.AssertionError("Can't get here."); } @@ -25972,21 +25925,6 @@ public final class AdminProtos { org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse.class, org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse.getDefaultInstance())); } - - public void getSpaceQuotaEnforcements( - org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller, - org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest request, - org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback done) { - channel.callMethod( - getDescriptor().getMethods().get(19), - controller, - request, - org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse.getDefaultInstance(), - org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcUtil.generalizeCallback( - done, - org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse.class, - org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse.getDefaultInstance())); - } } public static BlockingInterface newBlockingStub( @@ -26089,11 +26027,6 @@ public final class AdminProtos { org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller, org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest request) throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException; - - public org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse getSpaceQuotaEnforcements( - org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller, - org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest request) - throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException; } private static final class BlockingStub implements BlockingInterface { @@ -26330,18 +26263,6 @@ public final class AdminProtos { org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse.getDefaultInstance()); } - - public org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse getSpaceQuotaEnforcements( - org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller, - org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest request) - throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException { - return (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse) channel.callBlockingMethod( - getDescriptor().getMethods().get(19), - controller, - request, - org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse.getDefaultInstance()); - } - } // @@protoc_insertion_point(class_scope:hbase.pb.AdminService) @@ -26623,7 +26544,7 @@ public final class AdminProtos { "adRequest\022\'\n\ntable_name\030\001 \001(\0132\023.hbase.pb" + ".TableName\"C\n\025GetRegionLoadResponse\022*\n\014r" + "egion_loads\030\001 \003(\0132\024.hbase.pb.RegionLoad2" + - "\346\r\n\014AdminService\022P\n\rGetRegionInfo\022\036.hbas" + + "\360\014\n\014AdminService\022P\n\rGetRegionInfo\022\036.hbas" + "e.pb.GetRegionInfoRequest\032\037.hbase.pb.Get" + "RegionInfoResponse\022M\n\014GetStoreFile\022\035.hba", "se.pb.GetStoreFileRequest\032\036.hbase.pb.Get" + @@ -26664,12 +26585,9 @@ public final class AdminProtos { "GetRegionLoadResponse\022k\n\026GetSpaceQuotaSn" + "apshots\022\'.hbase.pb.GetSpaceQuotaSnapshot" + "sRequest\032(.hbase.pb.GetSpaceQuotaSnapsho" + - "tsResponse\022t\n\031GetSpaceQuotaEnforcements\022" + - "*.hbase.pb.GetSpaceQuotaEnforcementsRequ", - "est\032+.hbase.pb.GetSpaceQuotaEnforcements" + - "ResponseBH\n1org.apache.hadoop.hbase.shad" + - "ed.protobuf.generatedB\013AdminProtosH\001\210\001\001\240" + - "\001\001" + "tsResponseBH\n1org.apache.hadoop.hbase.sh" + + "aded.protobuf.generatedB\013AdminProtosH\001\210\001", + "\001\240\001\001" }; org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor. InternalDescriptorAssigner() { diff --git a/hbase-protocol-shaded/src/main/protobuf/Admin.proto b/hbase-protocol-shaded/src/main/protobuf/Admin.proto index 1a085e64b3..aa54e093df 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Admin.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Admin.proto @@ -319,8 +319,4 @@ service AdminService { /** Fetches the RegionServer's view of space quotas */ rpc GetSpaceQuotaSnapshots(GetSpaceQuotaSnapshotsRequest) returns(GetSpaceQuotaSnapshotsResponse); - - /** Fetches the RegionServer's space quota active enforcements */ - rpc GetSpaceQuotaEnforcements(GetSpaceQuotaEnforcementsRequest) - returns(GetSpaceQuotaEnforcementsResponse); } diff --git a/hbase-protocol-shaded/src/main/protobuf/Quota.proto b/hbase-protocol-shaded/src/main/protobuf/Quota.proto index 0d171b3c0b..0d7443524f 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Quota.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Quota.proto @@ -136,18 +136,6 @@ message GetSpaceQuotaSnapshotsResponse { repeated TableQuotaSnapshot snapshots = 1; } -message GetSpaceQuotaEnforcementsRequest { -} - -message GetSpaceQuotaEnforcementsResponse { - // Cannot use TableName as a map key, do the repeated nested message by hand. - message TableViolationPolicy { - optional TableName table_name = 1; - optional SpaceViolationPolicy violation_policy = 2; - } - repeated TableViolationPolicy violation_policies = 1; -} - message GetQuotaStatesRequest { } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 5dfccae61c..679bba2f97 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -195,13 +195,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType; import org.apache.hadoop.hbase.shaded.protobuf.generated.MapReduceProtos.ScanMetrics; -import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest; -import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse.TableQuotaSnapshot; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceViolationPolicy; -import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse.TableViolationPolicy; import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.RequestHeader; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescriptor; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; @@ -3311,31 +3308,4 @@ public class RSRpcServices implements HBaseRPCErrorHandler, throw new ServiceException(e); } } - - @Override - public GetSpaceQuotaEnforcementsResponse getSpaceQuotaEnforcements( - RpcController controller, GetSpaceQuotaEnforcementsRequest request) - throws ServiceException { - try { - final RegionServerSpaceQuotaManager manager = - regionServer.getRegionServerSpaceQuotaManager(); - final GetSpaceQuotaEnforcementsResponse.Builder builder = - GetSpaceQuotaEnforcementsResponse.newBuilder(); - if (manager != null) { - ActivePolicyEnforcement enforcements = manager.getActiveEnforcements(); - for (Entry enforcement - : enforcements.getPolicies().entrySet()) { - SpaceViolationPolicy pbPolicy = SpaceViolationPolicy.valueOf( - enforcement.getValue().getPolicyName()); - builder.addViolationPolicies(TableViolationPolicy.newBuilder() - .setTableName(ProtobufUtil.toProtoTableName(enforcement.getKey())) - .setViolationPolicy(pbPolicy).build()); - } - } - return builder.build(); - } catch (Exception e) { - throw new ServiceException(e); - } - } - } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java index e52114e4d5..bac2ee6e0a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java @@ -100,8 +100,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.PrepareBul import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.PrepareBulkLoadResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanResponse; -import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest; -import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; @@ -737,11 +735,4 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices { throws ServiceException { return null; } - - @Override - public GetSpaceQuotaEnforcementsResponse getSpaceQuotaEnforcements( - RpcController controller, GetSpaceQuotaEnforcementsRequest request) - throws ServiceException { - return null; - } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java index 2cd67c990d..2020e3cf41 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java @@ -186,12 +186,13 @@ public class TestQuotaStatusRPCs { } }); - Map violations = - QuotaTableUtil.getRegionServerQuotaViolations( - TEST_UTIL.getConnection(), rs.getServerName()); - SpaceViolationPolicy policy = violations.get(tn); - assertNotNull("Did not find policy for " + tn, policy); - assertEquals(SpaceViolationPolicy.NO_INSERTS, policy); + // We obtain the violations for a RegionServer by observing the snapshots + Map snapshots = + QuotaTableUtil.getRegionServerQuotaSnapshots(TEST_UTIL.getConnection(), rs.getServerName()); + SpaceQuotaSnapshot snapshot = snapshots.get(tn); + assertNotNull("Did not find snapshot for " + tn, snapshot); + assertTrue(snapshot.getQuotaStatus().isInViolation()); + assertEquals(SpaceViolationPolicy.NO_INSERTS, snapshot.getQuotaStatus().getPolicy()); } @Test diff --git a/hbase-shell/src/main/ruby/hbase/quotas.rb b/hbase-shell/src/main/ruby/hbase/quotas.rb index a2b21fab17..784896e730 100644 --- a/hbase-shell/src/main/ruby/hbase/quotas.rb +++ b/hbase-shell/src/main/ruby/hbase/quotas.rb @@ -169,13 +169,21 @@ module Hbase QuotaTableUtil.getMasterReportedTableSizes(@admin.getConnection()) end - def get_rs_quota_snapshots(rs) - QuotaTableUtil.getRegionServerQuotaSnapshots(@admin.getConnection(), - ServerName.valueOf(rs)) + def get_quota_snapshots(regionserver=nil) + # Ask a regionserver if we were given one + return get_rs_quota_snapshots(regionserver) if regionserver + # Otherwise, read from the quota table + get_quota_snapshots_from_table end - def get_rs_quota_violations(rs) - QuotaTableUtil.getRegionServerQuotaViolations(@admin.getConnection(), + def get_quota_snapshots_from_table() + # Reads the snapshots from the hbase:quota table + QuotaTableUtil.getSnapshots(@admin.getConnection()) + end + + def get_rs_quota_snapshots(rs) + # Reads the snapshots from a specific regionserver + QuotaTableUtil.getRegionServerQuotaSnapshots(@admin.getConnection(), ServerName.valueOf(rs)) end diff --git a/hbase-shell/src/main/ruby/hbase_constants.rb b/hbase-shell/src/main/ruby/hbase_constants.rb index a857403932..fce75f9854 100644 --- a/hbase-shell/src/main/ruby/hbase_constants.rb +++ b/hbase-shell/src/main/ruby/hbase_constants.rb @@ -82,6 +82,7 @@ module HBaseConstants CONFIG = 'CONFIG' DATA = 'DATA' POLICY = 'POLICY' + REGIONSERVER = 'REGIONSERVER' # Load constants from hbase java API def self.promote_constants(constants) diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index 70b8599628..ef17af1b04 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -420,7 +420,6 @@ Shell.load_command_group( set_quota list_quotas list_quota_table_sizes - list_quota_violations list_quota_snapshots ] ) diff --git a/hbase-shell/src/main/ruby/shell/commands/list_quota_snapshots.rb b/hbase-shell/src/main/ruby/shell/commands/list_quota_snapshots.rb index c907762528..bec7f02ed4 100644 --- a/hbase-shell/src/main/ruby/shell/commands/list_quota_snapshots.rb +++ b/hbase-shell/src/main/ruby/shell/commands/list_quota_snapshots.rb @@ -22,25 +22,41 @@ module Shell class ListQuotaSnapshots < Command def help return <<-EOF -Lists the current snapshot of quotas on the given RegionServer. This -information filters to each RegionServer from the Master. For each -table, a snapshot includes the filesystem use, the filesystem limit, -and the policy to enact when the limit is exceeded. This command is -useful for debugging the running state of a cluster using filesystem quotas. +Lists the current space quota snapshots with optional selection criteria. +Snapshots encapsulate relevant information to space quotas such as space +use, configured limits, and quota violation details. This command is +useful for understanding the current state of a cluster with space quotas. + +By default, this command will read all snapshots stored in the system from +the hbase:quota table. A table name or namespace can be provided to filter +the snapshots returned. RegionServers maintain a copy of snapshots, refreshing +at a regular interval; by providing a RegionServer option, snapshots will +be retreived from that RegionServer instead of the quota table. For example: - hbase> list_quota_snapshots 'regionserver1.domain,16020,1483482894742' + hbase> list_quota_snapshots + hbase> list_quota_snapshots({TABLE => 'table1'}) + hbase> list_quota_snapshots({NAMESPACE => 'org1'}) + hbase> list_quota_snapshots({REGIONSERVER => 'server1.domain,16020,1483482894742'}) + hbase> list_quota_snapshots({NAMESPACE => 'org1', REGIONSERVER => 'server1.domain,16020,1483482894742'}) EOF end - def command(hostname, args = {}) - formatter.header(["TABLE", "USAGE", "LIMIT", "IN VIOLATION", "POLICY"]) + def command(args = {}) + # All arguments may be nil + desired_table = args[TABLE] + desired_namespace = args[NAMESPACE] + desired_regionserver = args[REGIONSERVER] + formatter.header(["TABLE", "USAGE", "LIMIT", "IN_VIOLATION", "POLICY"]) count = 0 - quotas_admin.get_rs_quota_snapshots(hostname).each do |tableName,snapshot| + quotas_admin.get_quota_snapshots(desired_regionserver).each do |table_name,snapshot| + # Skip this snapshot if it's for a table/namespace the user did not ask for + next unless accept? table_name, desired_table, desired_namespace status = snapshot.getQuotaStatus() policy = get_policy(status) - formatter.row([tableName.to_s, snapshot.getUsage().to_s, snapshot.getLimit().to_s, status.isInViolation().to_s, policy]) + formatter.row([table_name.to_s, snapshot.getUsage().to_s, snapshot.getLimit().to_s, + status.isInViolation().to_s, policy]) count += 1 end formatter.footer(count) @@ -54,6 +70,18 @@ EOF "None" end end + + def accept?(table_name, desired_table=nil, desired_namespace=nil) + # Check the table name if given one + if desired_table and table_name.getQualifierAsString() != desired_table + return false + end + # Check the namespace if given one + if desired_namespace and table_name.getNamespaceAsString() != desired_namespace + return false + end + true + end end end end diff --git a/hbase-shell/src/main/ruby/shell/commands/list_quota_table_sizes.rb b/hbase-shell/src/main/ruby/shell/commands/list_quota_table_sizes.rb index 9325477e89..53395436a5 100644 --- a/hbase-shell/src/main/ruby/shell/commands/list_quota_table_sizes.rb +++ b/hbase-shell/src/main/ruby/shell/commands/list_quota_table_sizes.rb @@ -22,10 +22,10 @@ module Shell class ListQuotaTableSizes < Command def help return <<-EOF -Lists the sizes of the tables in HBase as collected -for the purpose of implementing filesystem utilization -quotas. This information is extracted from the HBase -Master and drives future quota actions in the cluster. +Lists the computed size of each table in the cluster as computed by +all RegionServers. This is the raw information that the Master uses to +make decisions about space quotas. Most times, using `list_quota_snapshots` +provides a higher-level of insight than this command. For example: diff --git a/hbase-shell/src/main/ruby/shell/commands/list_quota_violations.rb b/hbase-shell/src/main/ruby/shell/commands/list_quota_violations.rb deleted file mode 100644 index f1836a2b5a..0000000000 --- a/hbase-shell/src/main/ruby/shell/commands/list_quota_violations.rb +++ /dev/null @@ -1,48 +0,0 @@ -# -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -module Shell - module Commands - class ListQuotaViolations < Command - def help - return <<-EOF -Lists the current quota violations being enforced by a RegionServer. -Violations are enacted based on the quota snapshot information a RegionServer -holds (see list_quota_snapshots). Each violation contains the action the -RegionServer is taking on the table. This command is useful in debugging -the running state of a cluster using filesystem quotas. - -For example: - - hbase> list_quota_violations 'regionserver1.domain,16020,1483482894742' -EOF - end - - def command(hostname, args = {}) - formatter.header(["TABLE", "POLICY"]) - count = 0 - quotas_admin.get_rs_quota_violations(hostname).each do |tableName,policy| - formatter.row([tableName.to_s, policy.name]) - count += 1 - end - formatter.footer(count) - end - end - end -end diff --git a/hbase-shell/src/test/ruby/hbase/quotas_test_no_cluster.rb b/hbase-shell/src/test/ruby/hbase/quotas_test_no_cluster.rb index 7504488503..7de122553f 100644 --- a/hbase-shell/src/test/ruby/hbase/quotas_test_no_cluster.rb +++ b/hbase-shell/src/test/ruby/hbase/quotas_test_no_cluster.rb @@ -25,6 +25,10 @@ require 'hbase/table' include HBaseConstants +java_import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot +java_import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy +java_import org.apache.hadoop.hbase.TableName + module Hbase class NoClusterSpaceQuotasTest < Test::Unit::TestCase include TestHelpers @@ -54,9 +58,6 @@ module Hbase end define_test 'get policy name for status not in violation' do - java_import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot - java_import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy - okStatus = SpaceQuotaSnapshot::SpaceQuotaStatus::notInViolation() # By default, statuses are in violation violatedStatus = SpaceQuotaSnapshot::SpaceQuotaStatus.new(SpaceViolationPolicy::NO_INSERTS) @@ -65,5 +66,17 @@ module Hbase assert_equal('None', quotaSnapshotCommand.get_policy(okStatus)) assert_equal('NO_INSERTS', quotaSnapshotCommand.get_policy(violatedStatus)) end + + define_test 'table and namespace filtering in list_quota_snapshots' do + cmd = ::Shell::Commands::ListQuotaSnapshots.new(nil) + assert cmd.accept?(TableName.valueOf('t1')) == true + assert cmd.accept?(TableName.valueOf('t1'), nil, nil) == true + assert cmd.accept?(TableName.valueOf('t1'), 't1', nil) == true + assert cmd.accept?(TableName.valueOf('t1'), 't2', nil) == false + assert cmd.accept?(TableName.valueOf('t1'), nil, 'ns1') == false + assert cmd.accept?(TableName.valueOf('ns1:t1'), nil, 'ns1') == true + assert cmd.accept?(TableName.valueOf('ns1:t1'), 't1', nil) == true + assert cmd.accept?(TableName.valueOf('ns1:t1'), 't1', 'ns1') == true + end end end -- 2.12.2