diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java index f52521b..a9a6cb8 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java @@ -92,6 +92,9 @@ public class QuotaSettingsFactory { if (quotas.getBypassGlobals() == true) { settings.add(new QuotaGlobalsSettingsBypass(userName, tableName, namespace, true)); } + if (quotas.hasMaxTables()) { + settings.add(new TableNumberSettings(userName, namespace, quotas.getMaxTables())); + } return settings; } @@ -265,4 +268,30 @@ public class QuotaSettingsFactory { public static QuotaSettings bypassGlobals(final String userName, final boolean bypassGlobals) { return new QuotaGlobalsSettingsBypass(userName, null, null, bypassGlobals); } + + /* ========================================================================== + * Table Number Settings + */ + + /** + * Set the "max table" for the specified user + * + * @param userName the user to limit + * @param maxTables the maximum number of tables allowed for the user + * @return the quota settings + */ + public static QuotaSettings userMaxTables(final String userName, final int maxTables) { + return new TableNumberSettings(userName, null, maxTables); + } + + /** + * Set the "max table" for the specified namespace + * + * @param namespace the namespace to limit + * @param maxTables the maximum number of tables allowed in the namespace + * @return the quota settings + */ + public static QuotaSettings namespaceMaxTables(final String namespace, final int maxTables) { + return new TableNumberSettings(null, namespace, maxTables); + } } \ No newline at end of file diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java index 0618bc2..21e3504 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.filter.RowFilter; import org.apache.hadoop.hbase.filter.RegexStringComparator; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas; +import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.QuotaUsage; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Strings; @@ -70,6 +71,7 @@ public class QuotaTableUtil { protected static final byte[] QUOTA_FAMILY_INFO = Bytes.toBytes("q"); protected static final byte[] QUOTA_FAMILY_USAGE = Bytes.toBytes("u"); + protected static final byte[] QUOTA_QUALIFIER_USAGE = Bytes.toBytes("u"); protected static final byte[] QUOTA_QUALIFIER_SETTINGS = Bytes.toBytes("s"); protected static final byte[] QUOTA_QUALIFIER_SETTINGS_PREFIX = Bytes.toBytes("s."); protected static final byte[] QUOTA_USER_ROW_KEY_PREFIX = Bytes.toBytes("u."); @@ -295,6 +297,25 @@ public class QuotaTableUtil { } /* ========================================================================= + * Quota "usage" helpers + */ + public static QuotaUsage getUserQuotaUsage(final Configuration conf, final String user) + throws IOException { + return getQuotaUsage(conf, getUserRowKey(user), QUOTA_QUALIFIER_USAGE); + } + + private static QuotaUsage getQuotaUsage(final Configuration conf, final byte[] rowKey, + final byte[] qualifier) throws IOException { + Get get = new Get(rowKey); + get.addColumn(QUOTA_FAMILY_USAGE, qualifier); + Result result = doGet(conf, get); + if (result.isEmpty()) { + return null; + } + return quotaUsageFromData(result.getValue(QUOTA_FAMILY_USAGE, qualifier)); + } + + /* ========================================================================= * Quotas protobuf helpers */ protected static Quotas quotasFromData(final byte[] data) throws IOException { @@ -316,9 +337,25 @@ public class QuotaTableUtil { boolean hasSettings = false; hasSettings |= quotas.hasThrottle(); hasSettings |= quotas.hasBypassGlobals(); + hasSettings |= quotas.hasMaxTables(); return !hasSettings; } + protected static QuotaUsage quotaUsageFromData(final byte[] data) throws IOException { + int magicLen = ProtobufUtil.lengthOfPBMagic(); + if (!ProtobufUtil.isPBMagicPrefix(data, 0, magicLen)) { + throw new IOException("Missing pb magic prefix"); + } + return QuotaUsage.parseFrom(new ByteArrayInputStream(data, magicLen, data.length - magicLen)); + } + + protected static byte[] quotaUsageToData(final QuotaUsage data) throws IOException { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + stream.write(ProtobufUtil.PB_MAGIC); + data.writeTo(stream); + return stream.toByteArray(); + } + /* ========================================================================= * HTable helpers */ diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaType.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaType.java index 9486d84..777229a 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaType.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaType.java @@ -29,4 +29,5 @@ import org.apache.hadoop.classification.InterfaceStability; public enum QuotaType { THROTTLE, GLOBAL_BYPASS, + TABLE_NUMBER, } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/TableNumberSettings.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/TableNumberSettings.java new file mode 100644 index 0000000..f170861 --- /dev/null +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/TableNumberSettings.java @@ -0,0 +1,54 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetQuotaRequest; + +@InterfaceAudience.Private +@InterfaceStability.Evolving +class TableNumberSettings extends QuotaSettings { + private final int maxTables; + + TableNumberSettings(final String userName, final String namespace, + final int maxTables) { + super(userName, null, namespace); + this.maxTables = maxTables; + } + + public int getMaxTables() { + return maxTables; + } + + @Override + public QuotaType getQuotaType() { + return QuotaType.TABLE_NUMBER; + } + + @Override + protected void setupSetQuotaRequest(SetQuotaRequest.Builder builder) { + builder.setMaxTables(maxTables); + } + + @Override + public String toString() { + return "TYPE => TABLE_NUMBER, LIMIT => " + maxTables; + } +} diff --git a/hbase-protocol/src/main/protobuf/Master.proto b/hbase-protocol/src/main/protobuf/Master.proto index 1b282bf..c848135 100644 --- a/hbase-protocol/src/main/protobuf/Master.proto +++ b/hbase-protocol/src/main/protobuf/Master.proto @@ -376,6 +376,7 @@ message SetQuotaRequest { optional bool remove_all = 5; optional bool bypass_globals = 6; optional ThrottleRequest throttle = 7; + optional int32 max_tables = 8; } message SetQuotaResponse { diff --git a/hbase-protocol/src/main/protobuf/Quota.proto b/hbase-protocol/src/main/protobuf/Quota.proto index 6ef15fe..1c0d219 100644 --- a/hbase-protocol/src/main/protobuf/Quota.proto +++ b/hbase-protocol/src/main/protobuf/Quota.proto @@ -66,8 +66,15 @@ enum QuotaType { message Quotas { optional bool bypass_globals = 1 [default = false]; + + // Throttle optional Throttle throttle = 2; + + // "Objects" Limits + optional uint32 max_tables = 3; } message QuotaUsage { + // "Objects" Usage + optional uint32 tables_created = 1 [default = 0]; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java index b9eaab3..6537b19 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java @@ -19,7 +19,9 @@ package org.apache.hadoop.hbase.quotas; import java.io.IOException; +import java.io.InterruptedIOException; import java.util.HashSet; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -36,11 +38,13 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.coprocessor.BaseMasterObserver; import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.ipc.RequestContext; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.handler.CreateTableHandler; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetQuotaRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetQuotaResponse; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas; +import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.QuotaUsage; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Throttle; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.TimedQuota; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.ThrottleRequest; @@ -48,6 +52,8 @@ import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.ThrottleType; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.QuotaScope; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.TimeUnit; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.security.UserGroupInformation; /** * Master Quota Manager. @@ -91,6 +97,10 @@ public class MasterQuotaManager { tableLocks = new NamedLock(); userLocks = new NamedLock(); + LOG.debug("Initializing quota master coprocessor"); + masterServices.getMasterCoprocessorHost().load(MasterQuotaObserver.class, + Coprocessor.PRIORITY_SYSTEM, masterServices.getConfiguration()); + enabled = true; } @@ -293,6 +303,7 @@ public class MasterQuotaManager { Quotas.Builder builder = (quotas != null) ? quotas.toBuilder() : Quotas.newBuilder(); if (req.hasThrottle()) applyThrottle(builder, req.getThrottle()); if (req.hasBypassGlobals()) applyBypassGlobals(builder, req.getBypassGlobals()); + if (req.hasMaxTables()) applyMaxTables(builder, req.getMaxTables()); // Submit new changes quotas = builder.build(); @@ -384,6 +395,14 @@ public class MasterQuotaManager { } } + private void applyMaxTables(final Quotas.Builder quotas, int maxTables) { + if (maxTables >= 0 && maxTables < Integer.MAX_VALUE) { + quotas.setMaxTables(maxTables); + } else { + quotas.clearMaxTables(); + } + } + private void validateTimedQuota(final TimedQuota timedQuota) throws IOException { if (timedQuota.getSoftLimit() < 1) { throw new DoNotRetryIOException(new UnsupportedOperationException( @@ -392,6 +411,116 @@ public class MasterQuotaManager { } /* ========================================================================== + * Master Coprocessor + */ + public static class MasterQuotaObserver extends BaseMasterObserver { + public MasterQuotaObserver() { + } + + @Override + public void preCreateTable(ObserverContext ctx, + HTableDescriptor desc, HRegionInfo[] regions) throws IOException { + MasterServices services = ctx.getEnvironment().getMasterServices(); + if (!services.isInitialized() || desc.getTableName().isSystemTable()) return; + + Configuration conf = services.getConfiguration(); + Set tables = services.getTableStateManager().getTables(); + String namespace = desc.getTableName().getNamespaceAsString(); + + // Verify namespace "maxTables" quota + // TODO-MAYBE: This can be cached, but since create is rare enough we can avoid that + Quotas quota = QuotaUtil.getNamespaceQuota(conf, namespace); + if (quota != null && quota.hasMaxTables()) { + int ntables = countNamespaceTables(tables, namespace); + if ((ntables + 1) > quota.getMaxTables()) { + throw new QuotaExceededException("The table " + desc.getTableName().getNameAsString() + + "cannot be created as it would exceed maximum number of tables allowed " + + " in the namespace ."); + } + } + + // Verify user "maxTables" quota + String userName = getTableCreator(services, desc); + if (userName != null) { + try { + services.getMasterQuotaManager().userLocks.lock(userName); + try { + quota = QuotaUtil.getUserQuota(conf, userName); + if (quota != null && quota.hasMaxTables()) { + QuotaUsage usage = QuotaUtil.getUserQuotaUsage(conf, userName); + if ((usage.getTablesCreated() + 1) > quota.getMaxTables()) { + throw new QuotaExceededException("The table " + desc.getTableName().getNameAsString() + + "cannot be created as it would exceed maximum number of tables allowed " + + " for the user."); + } + + // TODO: This should be in postCreate.. but... + usage = incTableCreated(usage, 1); + QuotaUtil.addUserQuotaUsage(conf, userName, usage); + } + } finally { + services.getMasterQuotaManager().userLocks.unlock(userName); + } + } catch (InterruptedException e) { + IOException iie = new InterruptedIOException(); + iie.initCause(e); + throw iie; + } + } + } + + @Override + public void postDeleteTable(ObserverContext ctx, + TableName tableName) throws IOException { + MasterServices services = ctx.getEnvironment().getMasterServices(); + Configuration conf = services.getConfiguration(); + String userName = getTableCreator(services, tableName); + if (userName != null) { + try { + services.getMasterQuotaManager().userLocks.lock(userName); + try { + QuotaUsage usage = QuotaUtil.getUserQuotaUsage(conf, userName); + usage = incTableCreated(usage, -1); + QuotaUtil.addUserQuotaUsage(conf, userName, usage); + } finally { + services.getMasterQuotaManager().userLocks.unlock(userName); + } + } catch (InterruptedException e) { + IOException iie = new InterruptedIOException(); + iie.initCause(e); + throw iie; + } + } + } + + private static int countNamespaceTables(final Set tables, final String namespace) { + int count = 0; + for (TableName table: tables) { + if (namespace.equals(table.getNamespaceAsString())) { + count++; + } + } + return count; + } + + private String getTableCreator(MasterServices services, HTableDescriptor htd) throws IOException { + // TODO: Replace with HBASE-11996 + return htd.getOwnerString(); + } + + private String getTableCreator(MasterServices services, TableName table) throws IOException { + // TODO: Replace with HBASE-11996 + return services.getTableDescriptors().get(table).getOwnerString(); + } + + private QuotaUsage incTableCreated(QuotaUsage usage, int inc) { + QuotaUsage.Builder builder = usage != null ? usage.toBuilder() : QuotaUsage.newBuilder(); + builder.setTablesCreated(Math.max(0, builder.getTablesCreated() + inc)); + return builder.build(); + } + } + + /* ========================================================================== * Helpers */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java index 1c41c3d..9d0d40a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas; +import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.QuotaUsage; import org.apache.hadoop.hbase.regionserver.BloomType; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -264,6 +265,21 @@ public class QuotaUtil extends QuotaTableUtil { } /* ========================================================================= + * Quota "usage" helpers + */ + public static void addUserQuotaUsage(final Configuration conf, final String user, + final QuotaUsage data) throws IOException { + addQuotaUsage(conf, getUserRowKey(user), QUOTA_QUALIFIER_USAGE, data); + } + + private static void addQuotaUsage(final Configuration conf, final byte[] rowKey, + final byte[] qualifier, final QuotaUsage data) throws IOException { + Put put = new Put(rowKey); + put.add(QUOTA_FAMILY_USAGE, qualifier, quotaUsageToData(data)); + doPut(conf, put); + } + + /* ========================================================================= * HTable helpers */ private static void doPut(final Configuration conf, final Put put) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaAdmin.java index 622cac2..e141e68 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaAdmin.java @@ -79,14 +79,17 @@ public class TestQuotaAdmin { public void testSimpleScan() throws Exception { Admin admin = TEST_UTIL.getHBaseAdmin(); String userName = User.getCurrent().getShortName(); + String namespace = "NS0"; admin.setQuota(QuotaSettingsFactory .throttleUser(userName, ThrottleType.REQUEST_NUMBER, 6, TimeUnit.MINUTES)); admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, true)); + admin.setQuota(QuotaSettingsFactory.namespaceMaxTables(namespace, 10)); QuotaRetriever scanner = QuotaRetriever.open(TEST_UTIL.getConfiguration()); try { int countThrottle = 0; + int countTableNumber = 0; int countGlobalBypass = 0; for (QuotaSettings settings: scanner) { LOG.debug(settings); @@ -103,19 +106,25 @@ public class TestQuotaAdmin { case GLOBAL_BYPASS: countGlobalBypass++; break; + case TABLE_NUMBER: + countTableNumber++; + break; default: fail("unexpected settings type: " + settings.getQuotaType()); } } assertEquals(1, countThrottle); + assertEquals(1, countTableNumber); assertEquals(1, countGlobalBypass); } finally { scanner.close(); } admin.setQuota(QuotaSettingsFactory.unthrottleUser(userName)); - assertNumResults(1, null); + assertNumResults(2, null); admin.setQuota(QuotaSettingsFactory.bypassGlobals(userName, false)); + assertNumResults(1, null); + admin.setQuota(QuotaSettingsFactory.namespaceMaxTables(namespace, Integer.MAX_VALUE)); assertNumResults(0, null); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaMaxTables.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaMaxTables.java new file mode 100644 index 0000000..24f0cec --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaMaxTables.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.quotas; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.HTable; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.util.Bytes; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; + +@Category({MasterTests.class, MediumTests.class}) +public class TestQuotaMaxTables { + final Log LOG = LogFactory.getLog(getClass()); + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + + private final static byte[] FAMILY = Bytes.toBytes("cf"); + private final static byte[] QUALIFIER = Bytes.toBytes("q"); + private final static String NAMESPACE_DEFAULT = "default"; + private final static String NAMESPACE_TEST = "testQuota"; + + private final static String[] TABLE_NAMES = new String[] { + "TestQuotaMaxTables0", + "TestQuotaMaxTables1", + "TestQuotaMaxTables2", + }; + + private static HTable[] tables; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + TEST_UTIL.getConfiguration().setInt(QuotaCache.REFRESH_CONF_KEY, 2000); + TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10); + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); + TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); + TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); + TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true); + TEST_UTIL.startMiniCluster(1); + TEST_UTIL.waitTableAvailable(QuotaTableUtil.QUOTA_TABLE_NAME.getName()); + + TEST_UTIL.getHBaseAdmin().createNamespace(NamespaceDescriptor.create(NAMESPACE_TEST).build()); + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test(timeout=90000) + public void testNamespaceLimit() throws Exception { + final Admin admin = TEST_UTIL.getHBaseAdmin(); + + // Unlimited table creations + assertEquals(TABLE_NAMES.length, createTables(NAMESPACE_DEFAULT)); + deleteAllTables(NAMESPACE_DEFAULT); + + // limit to 2 tables in this namespace + admin.setQuota(QuotaSettingsFactory.namespaceMaxTables(NAMESPACE_DEFAULT, 2)); + assertEquals(2, createTables(NAMESPACE_DEFAULT)); + deleteAllTables(NAMESPACE_DEFAULT); + + // Unlimited table creation on test namespace + assertEquals(TABLE_NAMES.length, createTables(NAMESPACE_TEST)); + deleteAllTables(NAMESPACE_TEST); + + // unlimit the namespace max + admin.setQuota(QuotaSettingsFactory.namespaceMaxTables(NAMESPACE_DEFAULT, Integer.MAX_VALUE)); + assertEquals(TABLE_NAMES.length, createTables(NAMESPACE_DEFAULT)); + deleteAllTables(NAMESPACE_DEFAULT); + } + + @Test(timeout=90000) + public void testUserLimit() throws Exception { + final Admin admin = TEST_UTIL.getHBaseAdmin(); + final String userName = User.getCurrent().getShortName(); + + // Unlimited table creations + assertEquals(TABLE_NAMES.length, createTables(NAMESPACE_DEFAULT)); + deleteAllTables(NAMESPACE_DEFAULT); + + // limit to 2 tables for this user + admin.setQuota(QuotaSettingsFactory.userMaxTables(userName, 2)); + assertEquals(2, createTables(NAMESPACE_DEFAULT)); + assertEquals(0, createTables(NAMESPACE_TEST)); + deleteAllTables(NAMESPACE_DEFAULT); + + // unlimit the namespace max + admin.setQuota(QuotaSettingsFactory.userMaxTables(userName, Integer.MAX_VALUE)); + assertEquals(TABLE_NAMES.length, createTables(NAMESPACE_DEFAULT)); + deleteAllTables(NAMESPACE_DEFAULT); + } + + private int createTables(String namespace) throws IOException { + final Admin admin = TEST_UTIL.getHBaseAdmin(); + int count = 0; + try { + for (String tableName: TABLE_NAMES) { + HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(namespace, tableName)); + desc.setOwnerString(User.getCurrent().getShortName()); + desc.addFamily(new HColumnDescriptor(FAMILY)); + admin.createTable(desc); + count++; + } + } catch (QuotaExceededException e) { + LOG.info("Unable to create table " + (count + 1), e); + } + return count; + } + + private void deleteAllTables(String namespace) throws IOException { + for (String tableName: TABLE_NAMES) { + TEST_UTIL.deleteTableIfAny(TableName.valueOf(namespace, tableName)); + } + } +} diff --git a/hbase-shell/src/main/ruby/hbase/quotas.rb b/hbase-shell/src/main/ruby/hbase/quotas.rb index 3bcba26..c876e9b 100644 --- a/hbase-shell/src/main/ruby/hbase/quotas.rb +++ b/hbase-shell/src/main/ruby/hbase/quotas.rb @@ -28,6 +28,7 @@ java_import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory module HBaseQuotasConstants GLOBAL_BYPASS = 'GLOBAL_BYPASS' THROTTLE_TYPE = 'THROTTLE_TYPE' + TABLE_NUMBER = 'TABLE_NUMBER' THROTTLE = 'THROTTLE' REQUEST = 'REQUEST' end @@ -115,6 +116,29 @@ module Hbase @admin.setQuota(settings) end + def set_max_tables(args) + raise(ArgumentError, "Arguments should be a Hash") unless args.kind_of?(Hash) + + limit = args.delete(LIMIT) + if limit.eql? NONE + limit = 0x7fffffff + end + + if args.has_key?(USER) + user = args.delete(USER) + raise(ArgumentError, "Unexpected arguments: " + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.userMaxTables(user, limit) + elsif args.has_key?(NAMESPACE) + namespace = args.delete(NAMESPACE) + raise(ArgumentError, "Unexpected arguments: " + args.inspect) unless args.empty? + settings = QuotaSettingsFactory.namespaceMaxTables(namespace, limit) + else + raise "Expected USER or NAMESPACE" + end + + @admin.setQuota(settings) + end + def list_quotas(args = {}) raise(ArgumentError, "Arguments should be a Hash") unless args.kind_of?(Hash) diff --git a/hbase-shell/src/main/ruby/shell/commands/set_quota.rb b/hbase-shell/src/main/ruby/shell/commands/set_quota.rb index 40e8a10..cfbc1d4 100644 --- a/hbase-shell/src/main/ruby/shell/commands/set_quota.rb +++ b/hbase-shell/src/main/ruby/shell/commands/set_quota.rb @@ -32,6 +32,9 @@ with (B, K, M, G, T, P) as valid size unit and (sec, min, hour, day) as valid ti Currently the throttle limit is per machine - a limit of 100req/min means that each machine can execute 100req/min. +TYPE => TABLE_NUMBER +The maximum number of tables allowed for a particular user or a namespace. + For example: hbase> set_quota TYPE => THROTTLE, USER => 'u1', LIMIT => '10req/sec' @@ -41,7 +44,11 @@ For example: hbase> set_quota TYPE => THROTTLE, NAMESPACE => 'ns1', LIMIT => '10req/sec' hbase> set_quota TYPE => THROTTLE, TABLE => 't1', LIMIT => '10M/sec' hbase> set_quota TYPE => THROTTLE, USER => 'u1', LIMIT => NONE + hbase> set_quota USER => 'u1', GLOBAL_BYPASS => true + + hbase> set_quota TYPE => TABLE_NUMBER, USER => 'u1', LIMIT => 10 + hbase> set_quota TYPE => TABLE_NUMBER, NAMESPACE => 'ns1', LIMIT => 20 EOF end @@ -56,6 +63,8 @@ EOF else quotas_admin.throttle(args) end + when TABLE_NUMBER + quotas_admin.set_max_tables(args) else raise "Invalid TYPE argument. got " + qtype end