From d4f284170d0306fca6240c03096fe462d5f782c0 Mon Sep 17 00:00:00 2001 From: Ashish Singhi Date: Wed, 29 Apr 2015 19:11:58 +0530 Subject: [PATCH] HBASE-13593 Quota support for namespace should take restore and clone snapshot into account --- .../hbase/master/snapshot/SnapshotManager.java | 59 ++++++++-- .../hadoop/hbase/namespace/NamespaceAuditor.java | 14 +++ .../hbase/namespace/NamespaceStateManager.java | 26 +++++ .../hadoop/hbase/quotas/MasterQuotaManager.java | 6 + .../hbase/namespace/TestNamespaceAuditor.java | 129 +++++++++++++++++++++ 5 files changed, 227 insertions(+), 7 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java index 177ced2..a7e5172 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java @@ -44,8 +44,10 @@ import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.errorhandling.ForeignException; +import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.master.AssignmentManager; @@ -56,6 +58,7 @@ import org.apache.hadoop.hbase.master.MetricsMaster; import org.apache.hadoop.hbase.master.SnapshotSentinel; import org.apache.hadoop.hbase.master.cleaner.HFileCleaner; import org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner; +import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.procedure.MasterProcedureManager; import org.apache.hadoop.hbase.procedure.Procedure; import org.apache.hadoop.hbase.procedure.ProcedureCoordinator; @@ -730,11 +733,20 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable "."); } - // call coproc pre hook - if (cpHost != null) { - cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc); + // Table already exist. Check and update the region quota for this table namespace + checkAndUpdateNamespaceRegionQuota(fs, manifest, snapshotTableDesc, tableName); + try { + // call coproc pre hook + if (cpHost != null) { + cpHost.preRestoreSnapshot(reqSnapshot, snapshotTableDesc); + } + restoreSnapshot(snapshot, snapshotTableDesc); + } catch (IOException e) { + this.master.getMasterQuotaManager().removeTableFromNamespaceQuota(tableName); + LOG.error("Exception occurred while restoring the snapshot " + snapshot.getName() + + " as table " + tableName.getNameAsString(), e); + throw e; } - restoreSnapshot(snapshot, snapshotTableDesc); LOG.info("Restore snapshot=" + snapshot.getName() + " as table=" + tableName); if (cpHost != null) { @@ -742,10 +754,18 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable } } else { HTableDescriptor htd = RestoreSnapshotHelper.cloneTableSchema(snapshotTableDesc, tableName); - if (cpHost != null) { - cpHost.preCloneSnapshot(reqSnapshot, htd); + checkAndUpdateNamespaceQuota(snapshotTableDesc.getTableName(), tableName); + try { + if (cpHost != null) { + cpHost.preCloneSnapshot(reqSnapshot, htd); + } + cloneSnapshot(snapshot, htd); + } catch (IOException e) { + this.master.getMasterQuotaManager().removeTableFromNamespaceQuota(tableName); + LOG.error("Exception occurred while cloning the snapshot " + snapshot.getName() + + " as table " + tableName.getNameAsString(), e); + throw e; } - cloneSnapshot(snapshot, htd); LOG.info("Clone snapshot=" + snapshot.getName() + " as table=" + tableName); if (cpHost != null) { @@ -754,6 +774,31 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable } } + private void checkAndUpdateNamespaceQuota(TableName tableName, TableName cloneTableName) + throws IOException { + if (this.master.getMasterQuotaManager().isQuotaEnabled()) { + int splitKeysLength; + try (RegionLocator locator = this.master.getConnection().getRegionLocator(tableName)) { + splitKeysLength = locator.getStartKeys().length; + } + this.master.getMasterQuotaManager().checkNamespaceTableAndRegionQuota(cloneTableName, + splitKeysLength); + } + } + + private void checkAndUpdateNamespaceRegionQuota(FileSystem fs, SnapshotManifest manifest, + HTableDescriptor snapshotTableDesc, TableName cloneTableName) throws IOException { + if (this.master.getMasterQuotaManager().isQuotaEnabled()) { + RestoreSnapshotHelper restoreHelper = + new RestoreSnapshotHelper(master.getConfiguration(), fs, manifest, snapshotTableDesc, + rootDir, new ForeignExceptionDispatcher(), TaskMonitor.get().createStatus("")); + RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions(); + + this.master.getMasterQuotaManager().checkAndUpdateNamespaceRegionQuota(cloneTableName, + metaChanges.getRegionsToRestore().size()); + } + } + /** * Restore the specified snapshot. * The restore will fail if the destination table has a snapshot or restore in progress. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java index 6d6f143..1e737ca 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java @@ -79,6 +79,20 @@ public class NamespaceAuditor { } } + /** + * Check and update region count quota for an existing table. + * @param tName - table name for which region count to be updated. + * @param regions - Number of regions that will be added. + * @throws IOException Signals that an I/O exception has occurred. + */ + public void checkQuotaToUpdateRegion(TableName tName, int regions) throws IOException { + if (stateManager.isInitialized()) { + stateManager.checkAndUpdateNamespaceRegionCount(tName, regions); + } else { + checkTableTypeAndThrowException(tName); + } + } + private void checkTableTypeAndThrowException(TableName name) throws IOException { if (name.isSystemTable()) { LOG.debug("Namespace auditor checks not performed for table " + name.getNameAsString()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java index 2280e36..b9e37b6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java @@ -106,6 +106,32 @@ class NamespaceStateManager { return true; } + /** + * Check and update region count for an existing table. To handle scenarios like restore snapshot + * @param TableName name of the table for region count needs to be checked and updated + * @param incr count of regions + * @throws QuotaExceededException if quota exceeds for the number of regions allowed in a + * namespace + * @throws IOException Signals that an I/O exception has occurred. + */ + synchronized void checkAndUpdateNamespaceRegionCount(TableName name, int incr) throws IOException { + String namespace = name.getNamespaceAsString(); + NamespaceDescriptor nspdesc = getNamespaceDescriptor(namespace); + if (nspdesc != null) { + NamespaceTableAndRegionInfo currentStatus = getState(namespace); + int regionCountOfTable = currentStatus.getRegionCountOfTable(name); + if ((currentStatus.getRegionCount() - regionCountOfTable + incr) > TableNamespaceManager + .getMaxRegions(nspdesc)) { + throw new QuotaExceededException("The table " + name.getNameAsString() + + " region count cannot be updated as it would exceed maximum number " + + "of regions allowed in the namespace. The total number of regions permitted is " + + TableNamespaceManager.getMaxRegions(nspdesc)); + } + currentStatus.removeTable(name); + currentStatus.addTable(name, incr); + } + } + private NamespaceDescriptor getNamespaceDescriptor(String namespaceAsString) { try { return this.master.getNamespaceDescriptor(namespaceAsString); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java index 5fe5f8c..fd4844a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java @@ -313,6 +313,12 @@ public class MasterQuotaManager implements RegionStateListener { } } + public void checkAndUpdateNamespaceRegionQuota(TableName tName, int regions) throws IOException { + if (enabled) { + namespaceQuotaManager.checkQuotaToUpdateRegion(tName, regions); + } + } + public void onRegionMerged(HRegionInfo hri) throws IOException { if (enabled) { namespaceQuotaManager.updateQuotaForRegionMerge(hri); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java index cccdace..21a1196 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Mutation; +import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver; import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver; @@ -76,6 +77,7 @@ import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.RegionServerCoprocessorHost; +import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSUtils; @@ -673,4 +675,131 @@ public class TestNamespaceAuditor { ADMIN.createTable(tableDescOne); ADMIN.createTable(tableDescTwo, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4); } + + @Test(expected = QuotaExceededException.class, timeout = 30000) + public void testCloneSnapshotQuotaExceed() throws Exception { + String nsp = prefix + "_testTableQuotaExceedWithCloneSnapshot"; + NamespaceDescriptor nspDesc = + NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "1") + .build(); + ADMIN.createNamespace(nspDesc); + assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp)); + TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"); + TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"); + HTableDescriptor tableDescOne = new HTableDescriptor(tableName); + ADMIN.createTable(tableDescOne); + String snapshot = "snapshot_testTableQuotaExceedWithCloneSnapshot"; + ADMIN.snapshot(snapshot, tableName); + ADMIN.cloneSnapshot(snapshot, cloneTableName); + ADMIN.deleteSnapshot(snapshot); + } + + @Test(timeout = 180000) + public void testCloneSnapshot() throws Exception { + String nsp = prefix + "_testCloneSnapshot"; + NamespaceDescriptor nspDesc = + NamespaceDescriptor.create(nsp).addConfiguration(TableNamespaceManager.KEY_MAX_TABLES, "2") + .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "20").build(); + ADMIN.createNamespace(nspDesc); + assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp)); + TableName tableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"); + TableName cloneTableName = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table2"); + HTableDescriptor tableDescOne = new HTableDescriptor(tableName); + + ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4); + String snapshot = "snapshot_testCloneSnapshot"; + ADMIN.snapshot(snapshot, tableName); + ADMIN.cloneSnapshot(snapshot, cloneTableName); + + int tableLength; + try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) { + tableLength = locator.getStartKeys().length; + } + assertEquals(tableName.getNameAsString() + " should have four regions.", 4, tableLength); + + try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) { + tableLength = locator.getStartKeys().length; + } + assertEquals(cloneTableName.getNameAsString() + " should have four regions.", 4, tableLength); + + NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp); + assertEquals("Total tables count should be 2.", 2, nstate.getTables().size()); + assertEquals("Total regions count should be.", 8, nstate.getRegionCount()); + + ADMIN.deleteSnapshot(snapshot); + } + + @Test(timeout = 180000) + public void testRestoreSnapshot() throws Exception { + String nsp = prefix + "_testRestoreSnapshot"; + NamespaceDescriptor nspDesc = + NamespaceDescriptor.create(nsp) + .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build(); + ADMIN.createNamespace(nspDesc); + assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp)); + TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"); + HTableDescriptor tableDescOne = new HTableDescriptor(tableName1); + ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4); + + NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp); + assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount()); + + String snapshot = "snapshot_testRestoreSnapshot"; + ADMIN.snapshot(snapshot, tableName1); + + List regions = ADMIN.getTableRegions(tableName1); + Collections.sort(regions); + + ADMIN.split(tableName1, Bytes.toBytes("JJJ")); + Thread.sleep(2000); + assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount()); + + ADMIN.disableTable(tableName1); + ADMIN.restoreSnapshot(snapshot); + + assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount()); + + ADMIN.enableTable(tableName1); + ADMIN.deleteSnapshot(snapshot); + } + + @Test(timeout = 180000) + public void testRestoreSnapshotQuotaExceed() throws Exception { + String nsp = prefix + "_testRestoreSnapshotQuotaExceed"; + NamespaceDescriptor nspDesc = + NamespaceDescriptor.create(nsp) + .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10").build(); + ADMIN.createNamespace(nspDesc); + NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp); + assertNotNull("Namespace descriptor found null.", ndesc); + TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"); + HTableDescriptor tableDescOne = new HTableDescriptor(tableName1); + ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4); + + NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp); + assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount()); + + String snapshot = "snapshot_testRestoreSnapshotQuotaExceed"; + ADMIN.snapshot(snapshot, tableName1); + + List regions = ADMIN.getTableRegions(tableName1); + Collections.sort(regions); + + ADMIN.split(tableName1, Bytes.toBytes("JJJ")); + Thread.sleep(2000); + assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount()); + + ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2"); + ADMIN.modifyNamespace(ndesc); + + ADMIN.disableTable(tableName1); + try { + ADMIN.restoreSnapshot(snapshot); + fail("Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin" + + " wraps IOException into RestoreSnapshotException"); + } catch (RestoreSnapshotException ignore) { + } + ADMIN.enableTable(tableName1); + ADMIN.deleteSnapshot(snapshot); + } } -- 1.9.2.msysgit.0