diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 3a6ae59..c90e4ff 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -308,6 +308,9 @@ public class HMaster extends HRegionServer implements MasterServices, Server { // initialization may have not completed yet. volatile boolean serviceStarted = false; + // flag set after we complete asynchorized services and master initialization is done, + private final ProcedureEvent fullyInitialized = new ProcedureEvent("master fully initialized"); + // flag set after we complete assignMeta. private final ProcedureEvent serverCrashProcessingEnabled = new ProcedureEvent("server crash processing"); @@ -1045,7 +1048,26 @@ public class HMaster extends HRegionServer implements MasterServices, Server { void initNamespace() throws IOException { //create namespace manager tableNamespaceManager = new TableNamespaceManager(this); - tableNamespaceManager.start(); + + if (conf.getBoolean("hbase.master.start.waitfor.namespacemanager", false)) { + // If being asked not to async start namespace manager, then just block + // master service starting until namespace manager is ready. + tableNamespaceManager.start(); + return; + } + // else asynchronized start namespace manager + Threads.setDaemonThreadRunning(new Thread(new Runnable() { + @Override + public void run() { + // Start namespace manager and wait to it to be fully started. + try { + tableNamespaceManager.start(); + } catch (IOException e) { + LOG.error("Namespace manager failed to start. ", e); + abort("Shutdown Master due to namespace manager failed to start. ", e); + } + } + }, "Init Namespace Manager async")); } boolean isCatalogJanitorEnabled() { @@ -2553,6 +2575,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { throw new IOException("Table Namespace Manager not ready yet, try again later"); } } + /** * Report whether this master is currently the active master or not. * If not active master, we are parked on ZK waiting to become active. @@ -2598,6 +2621,15 @@ public class HMaster extends HRegionServer implements MasterServices, Server { return initialized; } + public ProcedureEvent getFullyInitializedEvent() { + if (!fullyInitialized.isReady() && + isInitialized() && + tableNamespaceManager.isTableNamespaceManagerStarted()) { + procedureExecutor.getEnvironment().setEventReady(fullyInitialized, true); + } + return fullyInitialized; + } + /** * ServerCrashProcessingEnabled is set false before completing assignMeta to prevent processing * of crashed servers. diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index f51a797..3e6bf5f 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -1564,7 +1564,7 @@ public class MasterRpcServices extends RSRpcServices @Override public SetQuotaResponse setQuota(RpcController c, SetQuotaRequest req) throws ServiceException { try { - master.checkInitialized(); + master.checkNamespaceManagerReady(); return master.getMasterQuotaManager().setQuota(req); } catch (Exception e) { throw new ServiceException(e); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java index 6bcfb77..0f0daf3 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java @@ -72,7 +72,7 @@ public class TableNamespaceManager { public static final String KEY_MAX_REGIONS = "hbase.namespace.quota.maxregions"; public static final String KEY_MAX_TABLES = "hbase.namespace.quota.maxtables"; static final String NS_INIT_TIMEOUT = "hbase.master.namespace.init.timeout"; - static final int DEFAULT_NS_INIT_TIMEOUT = 300000; + static final int DEFAULT_NS_INIT_TIMEOUT = 1800000; // default is 30 minutes public TableNamespaceManager(MasterServices masterServices) { this.masterServices = masterServices; @@ -93,7 +93,7 @@ public class TableNamespaceManager { long startTime = EnvironmentEdgeManager.currentTime(); int timeout = conf.getInt(NS_INIT_TIMEOUT, DEFAULT_NS_INIT_TIMEOUT); while (!isTableAvailableAndInitialized(false)) { - if (EnvironmentEdgeManager.currentTime() - startTime + 100 > timeout) { + if (EnvironmentEdgeManager.currentTime() - startTime > timeout) { // We can't do anything if ns is not online. throw new IOException("Timedout " + timeout + "ms waiting for namespace table to " + "be assigned"); @@ -105,6 +105,10 @@ public class TableNamespaceManager { } } + public boolean isTableNamespaceManagerStarted() { + return initialized; + } + private synchronized Table getNamespaceTable() throws IOException { if (!isTableNamespaceManagerInitialized()) { throw new IOException(this.getClass().getName() + " isn't ready to serve"); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java index e873156..cc3a59a 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java @@ -193,21 +193,20 @@ public class CreateNamespaceProcedure sb.append(")"); } - private boolean isBootstrapNamespace() { + private static boolean isBootstrapNamespace(final NamespaceDescriptor nsDescriptor) { return nsDescriptor.equals(NamespaceDescriptor.DEFAULT_NAMESPACE) || nsDescriptor.equals(NamespaceDescriptor.SYSTEM_NAMESPACE); } @Override protected boolean acquireLock(final MasterProcedureEnv env) { - if (!env.getMasterServices().isInitialized()) { - // Namespace manager might not be ready if master is not fully initialized, - // return false to reject user namespace creation; return true for default - // and system namespace creation (this is part of master initialization). - if (!isBootstrapNamespace() && env.waitInitialized(this)) { - return false; - } + // Namespace manager might not be ready if master is not fully initialized, + // return false to reject user namespace creation; return true for default + // and system namespace creation (this is part of master initialization). + if (!isBootstrapNamespace(nsDescriptor) && env.waitFullyInitialized(this)) { + return false; } + return env.getProcedureQueue().tryAcquireNamespaceExclusiveLock(this, getNamespaceName()); } @@ -331,7 +330,7 @@ public class CreateNamespaceProcedure protected static void setNamespaceQuota( final MasterProcedureEnv env, final NamespaceDescriptor nsDescriptor) throws IOException { - if (env.getMasterServices().isInitialized()) { + if (env.getMasterServices().isInitialized() && !isBootstrapNamespace(nsDescriptor)) { env.getMasterServices().getMasterQuotaManager().setNamespaceQuota(nsDescriptor); } } @@ -370,6 +369,6 @@ public class CreateNamespaceProcedure protected boolean shouldWaitClientAck(MasterProcedureEnv env) { // hbase and default namespaces are created on bootstrap internally by the system // the client does not know about this procedures. - return !isBootstrapNamespace(); + return !isBootstrapNamespace(nsDescriptor); } } \ No newline at end of file diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index 74433b4..7a927d5 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -269,7 +269,7 @@ public class CreateTableProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { - if (!getTableName().isSystemTable() && env.waitInitialized(this)) { + if (!getTableName().isSystemTable() && env.waitFullyInitialized(this)) { return false; } return env.getProcedureQueue().tryAcquireTableExclusiveLock(this, getTableName()); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java index 3911f54..7ec88d6 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java @@ -132,6 +132,10 @@ public class MasterProcedureEnv { return procSched.waitEvent(((HMaster)master).getInitializedEvent(), proc); } + public boolean waitFullyInitialized(Procedure proc) { + return procSched.waitEvent(((HMaster)master).getFullyInitializedEvent(), proc); + } + public boolean waitServerCrashProcessingEnabled(Procedure proc) { return procSched.waitEvent(((HMaster)master).getServerCrashProcessingEnabledEvent(), proc); } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestAddColumnFamilyProcedure.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestAddColumnFamilyProcedure.java index b3fa10a..ba9c475 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestAddColumnFamilyProcedure.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestAddColumnFamilyProcedure.java @@ -48,6 +48,7 @@ public class TestAddColumnFamilyProcedure { private static void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + conf.setBoolean("hbase.master.start.waitfor.namespacemanager", true); } @BeforeClass diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateNamespaceProcedure.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateNamespaceProcedure.java index e7ddff6..420f64a 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateNamespaceProcedure.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateNamespaceProcedure.java @@ -52,6 +52,7 @@ public class TestCreateNamespaceProcedure { private static void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + conf.setBoolean("hbase.master.start.waitfor.namespacemanager", true); } @BeforeClass diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java index 8860711..70c1b69 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateTableProcedure.java @@ -25,7 +25,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.ProcedureInfo; @@ -55,6 +54,7 @@ public class TestCreateTableProcedure { private static void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + conf.setBoolean("hbase.master.start.waitfor.namespacemanager", true); } @BeforeClass diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteColumnFamilyProcedure.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteColumnFamilyProcedure.java index 11d6f14..a015b74 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteColumnFamilyProcedure.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteColumnFamilyProcedure.java @@ -48,6 +48,7 @@ public class TestDeleteColumnFamilyProcedure { private static void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + conf.setBoolean("hbase.master.start.waitfor.namespacemanager", true); } @BeforeClass diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteNamespaceProcedure.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteNamespaceProcedure.java index b672519..6a0615e 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteNamespaceProcedure.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteNamespaceProcedure.java @@ -53,6 +53,7 @@ public class TestDeleteNamespaceProcedure { private static void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + conf.setBoolean("hbase.master.start.waitfor.namespacemanager", true); } @BeforeClass diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteTableProcedure.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteTableProcedure.java index 5ac0ea1..7362d7f 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteTableProcedure.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDeleteTableProcedure.java @@ -50,6 +50,7 @@ public class TestDeleteTableProcedure { private static void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + conf.setBoolean("hbase.master.start.waitfor.namespacemanager", true); } @BeforeClass diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDisableTableProcedure.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDisableTableProcedure.java index 72d8113..781d551 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDisableTableProcedure.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestDisableTableProcedure.java @@ -49,6 +49,7 @@ public class TestDisableTableProcedure { private static void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + conf.setBoolean("hbase.master.start.waitfor.namespacemanager", true); } @BeforeClass diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestEnableTableProcedure.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestEnableTableProcedure.java index baaa64d..0433136 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestEnableTableProcedure.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestEnableTableProcedure.java @@ -49,6 +49,7 @@ public class TestEnableTableProcedure { private static void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + conf.setBoolean("hbase.master.start.waitfor.namespacemanager", true); } @BeforeClass diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyColumnFamilyProcedure.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyColumnFamilyProcedure.java index 4ccf24b..f5be3da 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyColumnFamilyProcedure.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyColumnFamilyProcedure.java @@ -48,6 +48,7 @@ public class TestModifyColumnFamilyProcedure { private static void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + conf.setBoolean("hbase.master.start.waitfor.namespacemanager", true); } @BeforeClass diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java index 327afc4..271d482 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java @@ -48,6 +48,7 @@ public class TestModifyNamespaceProcedure { private static void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + conf.setBoolean("hbase.master.start.waitfor.namespacemanager", true); } @BeforeClass diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedure.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedure.java index ac3b84a..cc68e76 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedure.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyTableProcedure.java @@ -51,6 +51,7 @@ public class TestModifyTableProcedure { private static void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + conf.setBoolean("hbase.master.start.waitfor.namespacemanager", true); } @BeforeClass diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java index 8797a4f..3ebe577 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateTableProcedure.java @@ -51,6 +51,7 @@ public class TestTruncateTableProcedure { private static void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + conf.setBoolean("hbase.master.start.waitfor.namespacemanager", true); } @BeforeClass