From 3758c4cbc0b3b20a99628e4a64e23cd0382ea921 Mon Sep 17 00:00:00 2001 From: shardul-cr7 Date: Wed, 21 Aug 2019 00:41:11 -0700 Subject: [PATCH] HBASE-22013 SpaceQuotas - getNumRegions() returning wrong number of regions due to region replicas Signed-off-by: Sakthi --- .../hbase/quotas/QuotaObserverChore.java | 4 ++ .../quotas/SpaceQuotaHelperForTests.java | 22 ++++++--- .../hadoop/hbase/quotas/TestSpaceQuotas.java | 47 +++++++++++++++++++ 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java index 92a149c070ea7a63a7b1e5169396c2fbda43f6fe..2b76cfdc236da8ee807aaa960d5968e7be89cf1b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.MetricsMaster; @@ -764,6 +765,9 @@ public class QuotaObserverChore extends ScheduledChore { List regions = this.conn.getAdmin().getRegions(table); if (regions == null) { return 0; + } else { + // Filter the region replicas if any and return the original number of regions for a table. + RegionReplicaUtil.removeNonDefaultRegions(regions); } return regions.size(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/SpaceQuotaHelperForTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/SpaceQuotaHelperForTests.java index 10920b0001e90fe854090aa256bea137490f42fd..185121715296f8908aaa949de97f0a9352efdc25 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/SpaceQuotaHelperForTests.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/SpaceQuotaHelperForTests.java @@ -37,9 +37,12 @@ import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter.Predicate; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HStore; import org.apache.hadoop.hbase.regionserver.HStoreFile; @@ -301,15 +304,16 @@ public class SpaceQuotaHelperForTests { } TableName createTableWithRegions(Admin admin, int numRegions) throws Exception { - return createTableWithRegions( - testUtil.getAdmin(), NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR, numRegions); + return createTableWithRegions(admin, NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR, numRegions, + 0); } TableName createTableWithRegions(String namespace, int numRegions) throws Exception { - return createTableWithRegions(testUtil.getAdmin(), namespace, numRegions); + return createTableWithRegions(testUtil.getAdmin(), namespace, numRegions, 0); } - TableName createTableWithRegions(Admin admin, String namespace, int numRegions) throws Exception { + TableName createTableWithRegions(Admin admin, String namespace, int numRegions, + int numberOfReplicas) throws Exception { final TableName tn = getNextTableName(namespace); // Delete the old table @@ -319,8 +323,14 @@ public class SpaceQuotaHelperForTests { } // Create the table - HTableDescriptor tableDesc = new HTableDescriptor(tn); - tableDesc.addFamily(new HColumnDescriptor(F1)); + TableDescriptor tableDesc; + if (numberOfReplicas > 0) { + tableDesc = TableDescriptorBuilder.newBuilder(tn).setRegionReplication(numberOfReplicas) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(F1)).build(); + } else { + tableDesc = TableDescriptorBuilder.newBuilder(tn) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(F1)).build(); + } if (numRegions == 1) { admin.createTable(tableDesc); } else { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotas.java index a3fc0c28c3329e869d9e9027ac71b6be924f5394..abd88c330c44db870197791752318f502c6f744b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotas.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotas.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotEnabledException; import org.apache.hadoop.hbase.client.Admin; @@ -492,6 +493,52 @@ public class TestSpaceQuotas { setQuotaLimit(NON_EXISTENT_TABLE, SpaceViolationPolicy.DISABLE, 2L); } + @Test + public void testSetQuotaWithRegionReplicaSingleRegion() throws Exception { + setQuotaAndVerfiyForRegionReplication(1, 2, SpaceViolationPolicy.NO_INSERTS); + setQuotaAndVerfiyForRegionReplication(1, 2, SpaceViolationPolicy.NO_WRITES); + setQuotaAndVerfiyForRegionReplication(1, 2, SpaceViolationPolicy.NO_WRITES_COMPACTIONS); + setQuotaAndVerfiyForRegionReplication(1, 2, SpaceViolationPolicy.DISABLE); + } + + @Test + public void testSetQuotaWithRegionReplicaMultipleRegion() throws Exception { + setQuotaAndVerfiyForRegionReplication(5, 3, SpaceViolationPolicy.NO_INSERTS); + setQuotaAndVerfiyForRegionReplication(6, 3, SpaceViolationPolicy.NO_WRITES); + setQuotaAndVerfiyForRegionReplication(6, 3, SpaceViolationPolicy.NO_WRITES_COMPACTIONS); + setQuotaAndVerfiyForRegionReplication(6, 3, SpaceViolationPolicy.DISABLE); + } + + @Test + public void testSetQuotaWithSingleRegionZeroRegionReplica() throws Exception { + setQuotaAndVerfiyForRegionReplication(1, 0, SpaceViolationPolicy.NO_INSERTS); + setQuotaAndVerfiyForRegionReplication(1, 0, SpaceViolationPolicy.NO_WRITES); + setQuotaAndVerfiyForRegionReplication(1, 0, SpaceViolationPolicy.NO_WRITES_COMPACTIONS); + setQuotaAndVerfiyForRegionReplication(1, 0, SpaceViolationPolicy.DISABLE); + } + + @Test + public void testSetQuotaWithMultipleRegionZeroRegionReplicas() throws Exception { + setQuotaAndVerfiyForRegionReplication(5, 0, SpaceViolationPolicy.NO_INSERTS); + setQuotaAndVerfiyForRegionReplication(6, 0, SpaceViolationPolicy.NO_WRITES); + setQuotaAndVerfiyForRegionReplication(6, 0, SpaceViolationPolicy.NO_WRITES_COMPACTIONS); + setQuotaAndVerfiyForRegionReplication(6, 0, SpaceViolationPolicy.DISABLE); + } + + private void setQuotaAndVerfiyForRegionReplication(int region, int replicatedRegion, + SpaceViolationPolicy policy) throws Exception { + TableName tn = helper.createTableWithRegions(TEST_UTIL.getAdmin(), + NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR, region, replicatedRegion); + setQuotaLimit(tn, policy, 5L); + helper.writeData(tn, 5L * SpaceQuotaHelperForTests.ONE_MEGABYTE); + Put p = new Put(Bytes.toBytes("to_reject")); + p.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), + Bytes.toBytes("reject")); + // Adding a sleep for 5 sec, so all the chores run and to void flakiness of the test. + Thread.sleep(5000); + verifyViolation(policy, tn, p); + } + public void setQuotaAndViolateNextSwitchPoliciesAndValidate(SpaceViolationPolicy policy1, SpaceViolationPolicy policy2) throws Exception { Put put = new Put(Bytes.toBytes("to_reject")); -- 2.20.1 (Apple Git-117)