From f156637107e0cfbd6359d221cb510bd8a7a30b93 Mon Sep 17 00:00:00 2001 From: Sakthi Date: Wed, 25 Jul 2018 22:44:44 -0700 Subject: [PATCH] HBASE-20885 Removed entry for RPC quota from hbase:quota when RPC quota is removed --- .../hadoop/hbase/quotas/ThrottleSettings.java | 7 + .../hbase/quotas/GlobalQuotaSettingsImpl.java | 6 +- .../apache/hadoop/hbase/quotas/TestQuotaAdmin.java | 172 ++++++++++++++++++++- 3 files changed, 182 insertions(+), 3 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottleSettings.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottleSettings.java index 8cee7cdb01c17047f240b7485112a04214b1f448..b97b559983a4f75f28fcd6eb2a992932012a082f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottleSettings.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/ThrottleSettings.java @@ -48,6 +48,13 @@ class ThrottleSettings extends QuotaSettings { return proto.hasTimedQuota() ? proto.getTimedQuota().getSoftLimit() : -1; } + /** + * Returns a copy of the internal state of this + */ + QuotaProtos.ThrottleRequest getProto() { + return proto.toBuilder().build(); + } + public TimeUnit getTimeUnit() { return proto.hasTimedQuota() ? ProtobufUtil.toTimeUnit(proto.getTimedQuota().getTimeUnit()) : null; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/GlobalQuotaSettingsImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/GlobalQuotaSettingsImpl.java index 3893d00b83189d5561dbb8ce8791ce912fbff94d..11f19ca815981e2e50f59e86029a7ecedcf98129 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/GlobalQuotaSettingsImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/GlobalQuotaSettingsImpl.java @@ -23,11 +23,13 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory.QuotaGlobalsSettingsBypass; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota; @@ -175,10 +177,10 @@ public class GlobalQuotaSettingsImpl extends GlobalQuotaSettings { break; } } else { - clearThrottleBuilder(throttleBuilder); + throttleBuilder = null; } } else { - clearThrottleBuilder(throttleBuilder); + throttleBuilder = null; } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaAdmin.java index 572fb102aa53729a96266dfaac64e1c6839414e5..fef6c27710640e9b2a4adc8a479d6f9482872927 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaAdmin.java @@ -368,7 +368,7 @@ public class TestQuotaAdmin { } @Test - public void testSetModifyRemoveQuota() throws Exception { + public void testSetModifyRemoveSpaceQuota() throws Exception { Admin admin = TEST_UTIL.getAdmin(); final TableName tn = TableName.valueOf("sq_table2"); final long originalSizeLimit = 1024L * 1024L * 1024L * 1024L * 5L; // 5TB @@ -453,6 +453,176 @@ public class TestQuotaAdmin { assertEquals(expected, countResults(filter)); } + @Test + public void testSetGetRemoveRPCQuota() throws Exception { + Admin admin = TEST_UTIL.getAdmin(); + final TableName tn = TableName.valueOf("sq_table1"); + QuotaSettings settings = QuotaSettingsFactory.throttleTable(tn, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS); + admin.setQuota(settings); + + // Verify the Quotas in the table + try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) { + ResultScanner scanner = quotaTable.getScanner(new Scan()); + try { + Result r = Iterables.getOnlyElement(scanner); + CellScanner cells = r.cellScanner(); + assertTrue("Expected to find a cell", cells.advance()); + assertRPCQuota(ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS, cells.current()); + } finally { + scanner.close(); + } + } + + // Verify we can retrieve it via the QuotaRetriever API + QuotaRetriever quotaScanner = QuotaRetriever.open(admin.getConfiguration()); + try { + assertRPCQuota(ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS, Iterables.getOnlyElement(quotaScanner)); + } finally { + quotaScanner.close(); + } + + // Now, remove the quota + QuotaSettings removeQuota = QuotaSettingsFactory.unthrottleTable(tn); + admin.setQuota(removeQuota); + + // Verify that the record doesn't exist in the table + try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) { + ResultScanner scanner = quotaTable.getScanner(new Scan()); + try { + assertNull("Did not expect to find a quota entry", scanner.next()); + } finally { + scanner.close(); + } + } + + // Verify that we can also not fetch it via the API + quotaScanner = QuotaRetriever.open(admin.getConfiguration()); + try { + assertNull("Did not expect to find a quota entry", quotaScanner.next()); + } finally { + quotaScanner.close(); + } + + } + + @Test + public void testSetModifyRemoveRPCQuota() throws Exception { + Admin admin = TEST_UTIL.getAdmin(); + final TableName tn = TableName.valueOf("sq_table1"); + QuotaSettings settings = QuotaSettingsFactory.throttleTable(tn, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS); + admin.setQuota(settings); + + // Verify the Quotas in the table + try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) { + ResultScanner scanner = quotaTable.getScanner(new Scan()); + try { + Result r = Iterables.getOnlyElement(scanner); + CellScanner cells = r.cellScanner(); + assertTrue("Expected to find a cell", cells.advance()); + assertRPCQuota(ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS, cells.current()); + } finally { + scanner.close(); + } + } + + // Verify we can retrieve it via the QuotaRetriever API + QuotaRetriever quotaScanner = QuotaRetriever.open(admin.getConfiguration()); + try { + assertRPCQuota(ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS, Iterables.getOnlyElement(quotaScanner)); + } finally { + quotaScanner.close(); + } + + // Setting a limit and time unit should be reflected + QuotaSettings newSettings = QuotaSettingsFactory.throttleTable(tn, ThrottleType.REQUEST_SIZE, 3L, TimeUnit.DAYS); + admin.setQuota(newSettings); + + // Verify the new Quotas in the table + try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) { + ResultScanner scanner = quotaTable.getScanner(new Scan()); + try { + Result r = Iterables.getOnlyElement(scanner); + CellScanner cells = r.cellScanner(); + assertTrue("Expected to find a cell", cells.advance()); + assertRPCQuota(ThrottleType.REQUEST_SIZE, 3L, TimeUnit.DAYS, cells.current()); + } finally { + scanner.close(); + } + } + + // Verify we can retrieve the new quota via the QuotaRetriever API + quotaScanner = QuotaRetriever.open(admin.getConfiguration()); + try { + assertRPCQuota(ThrottleType.REQUEST_SIZE, 3L, TimeUnit.DAYS, Iterables.getOnlyElement(quotaScanner)); + } finally { + quotaScanner.close(); + } + + // Now, remove the quota + QuotaSettings removeQuota = QuotaSettingsFactory.unthrottleTable(tn); + admin.setQuota(removeQuota); + + // Verify that the record doesn't exist in the table + try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) { + ResultScanner scanner = quotaTable.getScanner(new Scan()); + try { + assertNull("Did not expect to find a quota entry", scanner.next()); + } finally { + scanner.close(); + } + } + + // Verify that we can also not fetch it via the API + quotaScanner = QuotaRetriever.open(admin.getConfiguration()); + try { + assertNull("Did not expect to find a quota entry", quotaScanner.next()); + } finally { + quotaScanner.close(); + } + + } + + private void assertRPCQuota(ThrottleType type, long limit, TimeUnit tu, Cell cell) throws Exception { + Quotas q = QuotaTableUtil.quotasFromData( + cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); + assertTrue("Quota should have rpc quota defined", q.hasThrottle()); + + QuotaProtos.Throttle rpcQuota = q.getThrottle(); + QuotaProtos.TimedQuota t = null; + + if(type == ThrottleType.REQUEST_SIZE){ + assertTrue(rpcQuota.hasReqSize()); + t = rpcQuota.getReqSize(); + } else if(type == ThrottleType.READ_NUMBER){ + assertTrue(rpcQuota.hasReadNum()); + t = rpcQuota.getReadNum(); + } else if(type == ThrottleType.READ_SIZE){ + assertTrue(rpcQuota.hasReadSize()); + t = rpcQuota.getReadSize(); + } else if(type == ThrottleType.REQUEST_NUMBER){ + assertTrue(rpcQuota.hasReqNum()); + t = rpcQuota.getReqNum(); + } else if(type == ThrottleType.WRITE_NUMBER){ + assertTrue(rpcQuota.hasWriteNum()); + t = rpcQuota.getWriteNum(); + } else if(type == ThrottleType.WRITE_SIZE){ + assertTrue(rpcQuota.hasWriteSize()); + t = rpcQuota.getWriteSize(); + } + assertEquals(t.getSoftLimit(),limit); + assertEquals(t.getTimeUnit(),ProtobufUtil.toProtoTimeUnit(tu)); + } + + private void assertRPCQuota(ThrottleType type, long limit, TimeUnit tu, QuotaSettings actualSettings) throws Exception { + assertTrue("The actual QuotaSettings was not an instance of " + ThrottleSettings.class + + " but of " + actualSettings.getClass(), actualSettings instanceof ThrottleSettings); + QuotaProtos.ThrottleRequest throttleRequest = ((ThrottleSettings) actualSettings).getProto(); + assertEquals(limit, throttleRequest.getTimedQuota().getSoftLimit()); + assertEquals(ProtobufUtil.toProtoTimeUnit(tu), throttleRequest.getTimedQuota().getTimeUnit()); + assertEquals(ProtobufUtil.toProtoThrottleType(type), throttleRequest.getType()); + + } + private void assertSpaceQuota( long sizeLimit, SpaceViolationPolicy violationPolicy, Cell cell) throws Exception { Quotas q = QuotaTableUtil.quotasFromData( -- 2.15.1 (Apple Git-101)