diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index febbf63..3950286 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -156,6 +156,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CompressionTest; import org.apache.hadoop.hbase.util.ConfigUtil; import org.apache.hadoop.hbase.util.EncryptionTest; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.util.HasThread; @@ -309,6 +310,10 @@ public class HMaster extends HRegionServer implements MasterServices, Server { // initialization may have not completed yet. volatile boolean serviceStarted = false; + // flag set after we complete asynchorized services and master initialization is done, + private final ProcedureEvent namespaceManagerInitialized = + new ProcedureEvent("master namespace manager initialized"); + // flag set after we complete assignMeta. private final ProcedureEvent serverCrashProcessingEnabled = new ProcedureEvent("server crash processing"); @@ -1081,7 +1086,28 @@ public class HMaster extends HRegionServer implements MasterServices, Server { void initNamespace() throws IOException { //create namespace manager tableNamespaceManager = new TableNamespaceManager(this); - tableNamespaceManager.start(); + + if (conf.getBoolean("hbase.master.start.waitfor.namespacemanager", false)) { + // If being asked not to async start namespace manager, then just block + // master service starting until namespace manager is ready. + tableNamespaceManager.start(); + setNamespaceManagerInitializedEvent(true); + return; + } + // else asynchronized start namespace manager + Threads.setDaemonThreadRunning(new Thread(new Runnable() { + @Override + public void run() { + // Start namespace manager and wait to it to be fully started. + try { + tableNamespaceManager.start(); + setNamespaceManagerInitializedEvent(true); + } catch (IOException e) { + LOG.error("Namespace manager failed to start. ", e); + abort("Shutdown Master due to namespace manager failed to start. ", e); + } + } + }, "Init Namespace Manager async")); } boolean isCatalogJanitorEnabled() { @@ -2598,11 +2624,26 @@ public class HMaster extends HRegionServer implements MasterServices, Server { void checkNamespaceManagerReady() throws IOException { checkInitialized(); - if (tableNamespaceManager == null || - !tableNamespaceManager.isTableAvailableAndInitialized(true)) { + if (tableNamespaceManager == null) { throw new IOException("Table Namespace Manager not ready yet, try again later"); + } else if (!tableNamespaceManager.isTableAvailableAndInitialized(true)) { + try { + // Wait some time. + long startTime = EnvironmentEdgeManager.currentTime(); + int timeout = conf.getInt("hbase.master.namespace.waitforready", 30000); + do { + Thread.sleep(100); + if (tableNamespaceManager.isTableNamespaceManagerStarted()) { + return; + } + } while (EnvironmentEdgeManager.currentTime() - startTime < timeout); + } catch (InterruptedException e) { + throw (InterruptedIOException) new InterruptedIOException().initCause(e); + } + throw new IOException("Table Namespace Manager not fully initialized, try again later"); } } + /** * Report whether this master is currently the active master or not. * If not active master, we are parked on ZK waiting to become active. @@ -2648,6 +2689,16 @@ public class HMaster extends HRegionServer implements MasterServices, Server { return initialized; } + public void setNamespaceManagerInitializedEvent(boolean isNamespaceManagerInitialized) { + procedureExecutor.getEnvironment().setEventReady( + namespaceManagerInitialized, + isNamespaceManagerInitialized); + } + + public ProcedureEvent getNamespaceManagerInitializedEvent() { + return namespaceManagerInitialized; + } + /** * ServerCrashProcessingEnabled is set false before completing assignMeta to prevent processing * of crashed servers. diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index c678c86..b025e5b 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -1605,7 +1605,7 @@ public class MasterRpcServices extends RSRpcServices @Override public SetQuotaResponse setQuota(RpcController c, SetQuotaRequest req) throws ServiceException { try { - master.checkInitialized(); + master.checkNamespaceManagerReady(); return master.getMasterQuotaManager().setQuota(req); } catch (Exception e) { throw new ServiceException(e); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java index 6bcfb77..7222f86 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; /** @@ -72,7 +73,7 @@ public class TableNamespaceManager { public static final String KEY_MAX_REGIONS = "hbase.namespace.quota.maxregions"; public static final String KEY_MAX_TABLES = "hbase.namespace.quota.maxtables"; static final String NS_INIT_TIMEOUT = "hbase.master.namespace.init.timeout"; - static final int DEFAULT_NS_INIT_TIMEOUT = 300000; + static final int DEFAULT_NS_INIT_TIMEOUT = 1800000; // default is 30 minutes public TableNamespaceManager(MasterServices masterServices) { this.masterServices = masterServices; @@ -93,7 +94,7 @@ public class TableNamespaceManager { long startTime = EnvironmentEdgeManager.currentTime(); int timeout = conf.getInt(NS_INIT_TIMEOUT, DEFAULT_NS_INIT_TIMEOUT); while (!isTableAvailableAndInitialized(false)) { - if (EnvironmentEdgeManager.currentTime() - startTime + 100 > timeout) { + if (EnvironmentEdgeManager.currentTime() - startTime > timeout) { // We can't do anything if ns is not online. throw new IOException("Timedout " + timeout + "ms waiting for namespace table to " + "be assigned"); @@ -105,6 +106,11 @@ public class TableNamespaceManager { } } + @VisibleForTesting + public boolean isTableNamespaceManagerStarted() { + return initialized; + } + private synchronized Table getNamespaceTable() throws IOException { if (!isTableNamespaceManagerInitialized()) { throw new IOException(this.getClass().getName() + " isn't ready to serve"); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java index e873156..d19635c 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java @@ -193,21 +193,20 @@ public class CreateNamespaceProcedure sb.append(")"); } - private boolean isBootstrapNamespace() { + private static boolean isBootstrapNamespace(final NamespaceDescriptor nsDescriptor) { return nsDescriptor.equals(NamespaceDescriptor.DEFAULT_NAMESPACE) || nsDescriptor.equals(NamespaceDescriptor.SYSTEM_NAMESPACE); } @Override protected boolean acquireLock(final MasterProcedureEnv env) { - if (!env.getMasterServices().isInitialized()) { - // Namespace manager might not be ready if master is not fully initialized, - // return false to reject user namespace creation; return true for default - // and system namespace creation (this is part of master initialization). - if (!isBootstrapNamespace() && env.waitInitialized(this)) { - return false; - } + // Namespace manager might not be ready if master is not fully initialized, + // return false to reject user namespace creation; return true for default + // and system namespace creation (this is part of master initialization). + if (!isBootstrapNamespace(nsDescriptor) && env.waitNamespaceManagerInitialized(this)) { + return false; } + return env.getProcedureQueue().tryAcquireNamespaceExclusiveLock(this, getNamespaceName()); } @@ -331,7 +330,7 @@ public class CreateNamespaceProcedure protected static void setNamespaceQuota( final MasterProcedureEnv env, final NamespaceDescriptor nsDescriptor) throws IOException { - if (env.getMasterServices().isInitialized()) { + if (env.getMasterServices().isInitialized() && !isBootstrapNamespace(nsDescriptor)) { env.getMasterServices().getMasterQuotaManager().setNamespaceQuota(nsDescriptor); } } @@ -370,6 +369,6 @@ public class CreateNamespaceProcedure protected boolean shouldWaitClientAck(MasterProcedureEnv env) { // hbase and default namespaces are created on bootstrap internally by the system // the client does not know about this procedures. - return !isBootstrapNamespace(); + return !isBootstrapNamespace(nsDescriptor); } } \ No newline at end of file diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index 74433b4..905f5b1 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -269,7 +269,7 @@ public class CreateTableProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { - if (!getTableName().isSystemTable() && env.waitInitialized(this)) { + if (!getTableName().isSystemTable() && env.waitNamespaceManagerInitialized(this)) { return false; } return env.getProcedureQueue().tryAcquireTableExclusiveLock(this, getTableName()); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java index a7ebc30..e011f29 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java @@ -210,7 +210,9 @@ public class DeleteNamespaceProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { - if (env.waitInitialized(this)) return false; + if (env.waitNamespaceManagerInitialized(this)) { + return false; + } return env.getProcedureQueue().tryAcquireNamespaceExclusiveLock(this, getNamespaceName()); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java index 3911f54..75822c8 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java @@ -132,6 +132,10 @@ public class MasterProcedureEnv { return procSched.waitEvent(((HMaster)master).getInitializedEvent(), proc); } + public boolean waitNamespaceManagerInitialized(Procedure proc) { + return procSched.waitEvent(((HMaster)master).getNamespaceManagerInitializedEvent(), proc); + } + public boolean waitServerCrashProcessingEnabled(Procedure proc) { return procSched.waitEvent(((HMaster)master).getServerCrashProcessingEnabledEvent(), proc); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java index 5d9548b..5953032 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java @@ -191,7 +191,9 @@ public class ModifyNamespaceProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { - if (env.waitInitialized(this)) return false; + if (env.waitNamespaceManagerInitialized(this)) { + return false; + } return env.getProcedureQueue().tryAcquireNamespaceExclusiveLock(this, getNamespaceName()); } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 8151759..1f0a2b2 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -102,6 +102,7 @@ import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.visibility.VisibilityLabelsCache; import org.apache.hadoop.hbase.tool.Canary; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.JVMClusterUtil; @@ -1460,6 +1461,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { .setMaxVersions(numVersions); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc, startKey, endKey, numRegions); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1498,6 +1500,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { hcd.setBloomFilterType(BloomType.NONE); htd.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(htd, splitKeys); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are // assigned @@ -1514,6 +1517,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { */ public HTable createTable(HTableDescriptor htd, byte[][] splitRows) throws IOException { + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(htd, splitRows); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(htd.getTableName()); @@ -1568,6 +1572,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { hcd.setBloomFilterType(BloomType.NONE); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc); return new HTable(c, desc.getTableName()); } @@ -1590,6 +1595,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { .setMaxVersions(numVersions); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1614,6 +1620,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { .setMaxVersions(numVersions); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc); return new HTable(c, desc.getTableName()); } @@ -1688,6 +1695,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { HColumnDescriptor hcd = new HColumnDescriptor(family).setMaxVersions(numVersions); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc, splitKeys); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1740,6 +1748,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { .setBlocksize(blockSize); desc.addFamily(hcd); } + // Make sure that master is fully initialized before calling create table. + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1779,6 +1789,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { desc.addFamily(hcd); i++; } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1811,6 +1822,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor(family); desc.addFamily(hcd); + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc, splitRows); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1843,12 +1855,35 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { HColumnDescriptor hcd = new HColumnDescriptor(family); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc, splitRows); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(desc.getTableName()); return new HTable(getConfiguration(), desc.getTableName()); } + private void waitUntilTableNamespaceManagerStarted() throws IOException { + final int maxwait = 60000; + final long startTime = EnvironmentEdgeManager.currentTime(); + final HMaster master = getMiniHBaseCluster().getMaster(); + do { + if(master.getTableNamespaceManager().isTableNamespaceManagerStarted()) { + break; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new IOException("Interrupt while waiting for master namespace manager starting."); + } + } while (EnvironmentEdgeManager.currentTime() - startTime < maxwait); + + if(!master.getTableNamespaceManager().isTableNamespaceManagerStarted()) { + throw new IOException( + "Cannot continue testing due to master namespace manager not started after waiting " + + (EnvironmentEdgeManager.currentTime() - startTime) + " milliseconds"); + } + } + /** * Create an unmanaged WAL. Be sure to close it when you're through. */ diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRollbackFromClient.java hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRollbackFromClient.java index 9230f31..01717ff 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRollbackFromClient.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRollbackFromClient.java @@ -222,7 +222,7 @@ public class TestRollbackFromClient { HColumnDescriptor col = new HColumnDescriptor(FAMILY); col.setMaxVersions(versions); desc.addFamily(col); - TEST_UTIL.getHBaseAdmin().createTable(desc); + TEST_UTIL.createTable(tableName, FAMILY, versions); int expected; List cells; try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());