diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index feda7fd..f0ef3e0 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -156,6 +156,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CompressionTest; import org.apache.hadoop.hbase.util.ConfigUtil; import org.apache.hadoop.hbase.util.EncryptionTest; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.util.HasThread; @@ -309,6 +310,10 @@ public class HMaster extends HRegionServer implements MasterServices, Server { // initialization may have not completed yet. volatile boolean serviceStarted = false; + // flag set after we complete asynchorized services and master initialization is done, + private final ProcedureEvent namespaceManagerInitialized = + new ProcedureEvent("master namespace manager initialized"); + // flag set after we complete assignMeta. private final ProcedureEvent serverCrashProcessingEnabled = new ProcedureEvent("server crash processing"); @@ -874,8 +879,8 @@ public class HMaster extends HRegionServer implements MasterServices, Server { periodicDoMetricsChore = new PeriodicDoMetrics(msgInterval, this); getChoreService().scheduleChore(periodicDoMetricsChore); - status.setStatus("Starting namespace manager"); - initNamespace(); + status.setStatus("Starting namespace manager and quota manager"); + initNamespaceAndQuotaManager(); if (this.cpHost != null) { try { @@ -892,11 +897,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server { // Set master as 'initialized'. setInitialized(true); - assignmentManager.checkIfShouldMoveSystemRegionAsync(); - - status.setStatus("Starting quota manager"); - initQuotaManager(); - // assign the meta replicas Set EMPTY_SET = new HashSet(); int numReplicas = conf.getInt(HConstants.META_REPLICAS_NUM, @@ -928,12 +928,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server { zombieDetector.interrupt(); } - private void initQuotaManager() throws IOException { - quotaManager = new MasterQuotaManager(this); - this.assignmentManager.setRegionStateListener((RegionStateListener) quotaManager); - quotaManager.start(); - } - /** * Create a {@link ServerManager} instance. * @param master @@ -1080,10 +1074,58 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } } - void initNamespace() throws IOException { + /* + * The main purpose is to start namespace manager and quota manager async to + * unblock overall master initialization + * + * @throws IOException + */ + private void initNamespaceAndQuotaManager() throws IOException { //create namespace manager tableNamespaceManager = new TableNamespaceManager(this); - tableNamespaceManager.start(); + + //create quota manager + this.quotaManager = new MasterQuotaManager(this); + this.assignmentManager.setRegionStateListener((RegionStateListener)quotaManager); + + if (conf.getBoolean("hbase.master.start.wait.for.namespacemanager", false)) { + // If being asked not to async start namespace manager, then just block + // master service starting until namespace manager is ready. + // + // Note: Quota manager depends on namespace manager. Therefore, its starting + // method has to be in-sync with namespace manager. + LOG.info("Starting namespace manager and quota manager synchronously"); + + tableNamespaceManager.start(); + setNamespaceManagerInitializedEvent(true); + + quotaManager.start(); + } else { // else asynchronously start namespace manager and quota manager + LOG.info("Starting namespace manager and quota manager asynchronously"); + Threads.setDaemonThreadRunning(new Thread(new Runnable() { + @Override + public void run() { + // Start namespace manager and wait to it to be fully started. + try { + tableNamespaceManager.start(); + setNamespaceManagerInitializedEvent(true); + LOG.info("Namespace manager started."); + } catch (IOException e) { + LOG.error("Namespace manager failed to start. ", e); + abort("Shutdown Master due to namespace manager failed to start. ", e); + } + // Quota Manager depends on Namespace manager to be fully initialized. + try { + quotaManager.start(); + LOG.info("Quota manager started."); + } catch (IOException ie) { + LOG.error("Quota Manager failed to start. ", ie); + abort("Shutdown Master due to Quota Manager failure to start. ", ie); + } + } + }, "Init Namespace Manager and Quota Manager Async")); + } + assignmentManager.checkIfShouldMoveSystemRegionAsync(); } boolean isCatalogJanitorEnabled() { @@ -2615,11 +2657,26 @@ public class HMaster extends HRegionServer implements MasterServices, Server { void checkNamespaceManagerReady() throws IOException { checkInitialized(); - if (tableNamespaceManager == null || - !tableNamespaceManager.isTableAvailableAndInitialized(true)) { + if (tableNamespaceManager == null) { throw new IOException("Table Namespace Manager not ready yet, try again later"); + } else if (!tableNamespaceManager.isTableAvailableAndInitialized(true)) { + try { + // Wait some time. + long startTime = EnvironmentEdgeManager.currentTime(); + int timeout = conf.getInt("hbase.master.namespace.wait.for.ready", 30000); + while (!tableNamespaceManager.isTableNamespaceManagerStarted() && + EnvironmentEdgeManager.currentTime() - startTime < timeout) { + Thread.sleep(100); + } + } catch (InterruptedException e) { + throw (InterruptedIOException) new InterruptedIOException().initCause(e); + } + if (!tableNamespaceManager.isTableNamespaceManagerStarted()) { + throw new IOException("Table Namespace Manager not fully initialized, try again later"); + } } } + /** * Report whether this master is currently the active master or not. * If not active master, we are parked on ZK waiting to become active. @@ -2665,6 +2722,16 @@ public class HMaster extends HRegionServer implements MasterServices, Server { return initialized; } + public void setNamespaceManagerInitializedEvent(boolean isNamespaceManagerInitialized) { + procedureExecutor.getEnvironment().setEventReady( + namespaceManagerInitialized, + isNamespaceManagerInitialized); + } + + public ProcedureEvent getNamespaceManagerInitializedEvent() { + return namespaceManagerInitialized; + } + /** * ServerCrashProcessingEnabled is set false before completing assignMeta to prevent processing * of crashed servers. diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 0155e8a..c216995 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -1303,7 +1303,7 @@ public class MasterRpcServices extends RSRpcServices public RestoreSnapshotResponse restoreSnapshot(RpcController controller, RestoreSnapshotRequest request) throws ServiceException { try { - master.checkInitialized(); + master.checkNamespaceManagerReady(); master.snapshotManager.checkSnapshotSupport(); // ensure namespace exists @@ -1606,7 +1606,7 @@ public class MasterRpcServices extends RSRpcServices @Override public SetQuotaResponse setQuota(RpcController c, SetQuotaRequest req) throws ServiceException { try { - master.checkInitialized(); + master.checkNamespaceManagerReady(); return master.getMasterQuotaManager().setQuota(req); } catch (Exception e) { throw new ServiceException(e); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java index 6bcfb77..224cda0 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; /** @@ -72,7 +73,8 @@ public class TableNamespaceManager { public static final String KEY_MAX_REGIONS = "hbase.namespace.quota.maxregions"; public static final String KEY_MAX_TABLES = "hbase.namespace.quota.maxtables"; static final String NS_INIT_TIMEOUT = "hbase.master.namespace.init.timeout"; - static final int DEFAULT_NS_INIT_TIMEOUT = 300000; + static final int DEFAULT_NS_INIT_TIMEOUT = 1800000; // default is 30 minutes + private static final int WAIT_MESSAGE_TO_PRINTOUT = 300000; // print out message every 5 minutes public TableNamespaceManager(MasterServices masterServices) { this.masterServices = masterServices; @@ -91,12 +93,18 @@ public class TableNamespaceManager { // If timed out, we will move ahead without initializing it. // So that it should be initialized later on lazily. long startTime = EnvironmentEdgeManager.currentTime(); + long msgCount = 0; + long waitTime; int timeout = conf.getInt(NS_INIT_TIMEOUT, DEFAULT_NS_INIT_TIMEOUT); while (!isTableAvailableAndInitialized(false)) { - if (EnvironmentEdgeManager.currentTime() - startTime + 100 > timeout) { + waitTime = EnvironmentEdgeManager.currentTime() - startTime; + if (waitTime > timeout) { // We can't do anything if ns is not online. throw new IOException("Timedout " + timeout + "ms waiting for namespace table to " + "be assigned"); + } else if (waitTime > msgCount * WAIT_MESSAGE_TO_PRINTOUT) { + LOG.info("Waiting for namespace table to be online. Time waited = " + waitTime + " ms."); + msgCount++; } Thread.sleep(100); } @@ -105,6 +113,11 @@ public class TableNamespaceManager { } } + @VisibleForTesting + public boolean isTableNamespaceManagerStarted() { + return initialized; + } + private synchronized Table getNamespaceTable() throws IOException { if (!isTableNamespaceManagerInitialized()) { throw new IOException(this.getClass().getName() + " isn't ready to serve"); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java index e873156..d19635c 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java @@ -193,21 +193,20 @@ public class CreateNamespaceProcedure sb.append(")"); } - private boolean isBootstrapNamespace() { + private static boolean isBootstrapNamespace(final NamespaceDescriptor nsDescriptor) { return nsDescriptor.equals(NamespaceDescriptor.DEFAULT_NAMESPACE) || nsDescriptor.equals(NamespaceDescriptor.SYSTEM_NAMESPACE); } @Override protected boolean acquireLock(final MasterProcedureEnv env) { - if (!env.getMasterServices().isInitialized()) { - // Namespace manager might not be ready if master is not fully initialized, - // return false to reject user namespace creation; return true for default - // and system namespace creation (this is part of master initialization). - if (!isBootstrapNamespace() && env.waitInitialized(this)) { - return false; - } + // Namespace manager might not be ready if master is not fully initialized, + // return false to reject user namespace creation; return true for default + // and system namespace creation (this is part of master initialization). + if (!isBootstrapNamespace(nsDescriptor) && env.waitNamespaceManagerInitialized(this)) { + return false; } + return env.getProcedureQueue().tryAcquireNamespaceExclusiveLock(this, getNamespaceName()); } @@ -331,7 +330,7 @@ public class CreateNamespaceProcedure protected static void setNamespaceQuota( final MasterProcedureEnv env, final NamespaceDescriptor nsDescriptor) throws IOException { - if (env.getMasterServices().isInitialized()) { + if (env.getMasterServices().isInitialized() && !isBootstrapNamespace(nsDescriptor)) { env.getMasterServices().getMasterQuotaManager().setNamespaceQuota(nsDescriptor); } } @@ -370,6 +369,6 @@ public class CreateNamespaceProcedure protected boolean shouldWaitClientAck(MasterProcedureEnv env) { // hbase and default namespaces are created on bootstrap internally by the system // the client does not know about this procedures. - return !isBootstrapNamespace(); + return !isBootstrapNamespace(nsDescriptor); } } \ No newline at end of file diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index 74433b4..905f5b1 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -269,7 +269,7 @@ public class CreateTableProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { - if (!getTableName().isSystemTable() && env.waitInitialized(this)) { + if (!getTableName().isSystemTable() && env.waitNamespaceManagerInitialized(this)) { return false; } return env.getProcedureQueue().tryAcquireTableExclusiveLock(this, getTableName()); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java index a7ebc30..e011f29 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java @@ -210,7 +210,9 @@ public class DeleteNamespaceProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { - if (env.waitInitialized(this)) return false; + if (env.waitNamespaceManagerInitialized(this)) { + return false; + } return env.getProcedureQueue().tryAcquireNamespaceExclusiveLock(this, getNamespaceName()); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java index 3911f54..75822c8 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java @@ -132,6 +132,10 @@ public class MasterProcedureEnv { return procSched.waitEvent(((HMaster)master).getInitializedEvent(), proc); } + public boolean waitNamespaceManagerInitialized(Procedure proc) { + return procSched.waitEvent(((HMaster)master).getNamespaceManagerInitializedEvent(), proc); + } + public boolean waitServerCrashProcessingEnabled(Procedure proc) { return procSched.waitEvent(((HMaster)master).getServerCrashProcessingEnabledEvent(), proc); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java index 5d9548b..5953032 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java @@ -191,7 +191,9 @@ public class ModifyNamespaceProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { - if (env.waitInitialized(this)) return false; + if (env.waitNamespaceManagerInitialized(this)) { + return false; + } return env.getProcedureQueue().tryAcquireNamespaceExclusiveLock(this, getNamespaceName()); } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index f60be66..c75d4bc 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -105,6 +105,7 @@ import org.apache.hadoop.hbase.security.HBaseKerberosUtils; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.visibility.VisibilityLabelsCache; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.JVMClusterUtil; @@ -1459,6 +1460,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { .setMaxVersions(numVersions); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc, startKey, endKey, numRegions); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1497,6 +1499,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { hcd.setBloomFilterType(BloomType.NONE); htd.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(htd, splitKeys); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are // assigned @@ -1513,6 +1516,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { */ public HTable createTable(HTableDescriptor htd, byte[][] splitRows) throws IOException { + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(htd, splitRows); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(htd.getTableName()); @@ -1567,6 +1571,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { hcd.setBloomFilterType(BloomType.NONE); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc); return new HTable(c, desc.getTableName()); } @@ -1589,6 +1594,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { .setMaxVersions(numVersions); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1613,6 +1619,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { .setMaxVersions(numVersions); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc); return new HTable(c, desc.getTableName()); } @@ -1687,6 +1694,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { HColumnDescriptor hcd = new HColumnDescriptor(family).setMaxVersions(numVersions); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc, splitKeys); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1739,6 +1747,8 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { .setBlocksize(blockSize); desc.addFamily(hcd); } + // Make sure that master is fully initialized before calling create table. + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1778,6 +1788,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { desc.addFamily(hcd); i++; } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1810,6 +1821,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor(family); desc.addFamily(hcd); + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc, splitRows); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1842,12 +1854,35 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { HColumnDescriptor hcd = new HColumnDescriptor(family); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc, splitRows); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(desc.getTableName()); return new HTable(getConfiguration(), desc.getTableName()); } + public void waitUntilTableNamespaceManagerStarted() throws IOException { + final int maxwait = 60000; + final long startTime = EnvironmentEdgeManager.currentTime(); + final HMaster master = getMiniHBaseCluster().getMaster(); + do { + if(master.getTableNamespaceManager().isTableNamespaceManagerStarted()) { + break; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new IOException("Interrupt while waiting for master namespace manager starting."); + } + } while (EnvironmentEdgeManager.currentTime() - startTime < maxwait); + + if(!master.getTableNamespaceManager().isTableNamespaceManagerStarted()) { + throw new IOException( + "Cannot continue testing due to master namespace manager not started after waiting " + + (EnvironmentEdgeManager.currentTime() - startTime) + " milliseconds"); + } + } + /** * Create an unmanaged WAL. Be sure to close it when you're through. */ diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRollbackFromClient.java hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRollbackFromClient.java index 9230f31..01717ff 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRollbackFromClient.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRollbackFromClient.java @@ -222,7 +222,7 @@ public class TestRollbackFromClient { HColumnDescriptor col = new HColumnDescriptor(FAMILY); col.setMaxVersions(versions); desc.addFamily(col); - TEST_UTIL.getHBaseAdmin().createTable(desc); + TEST_UTIL.createTable(tableName, FAMILY, versions); int expected; List cells; try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java index 292ec72..eb0af08 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java @@ -508,6 +508,9 @@ public class TestMasterFailover { HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); Configuration conf = TEST_UTIL.getConfiguration(); conf.setBoolean("hbase.assignment.usezk", true); + // The test depends on namespace region is online, therefore, we have to + // wait for namespace manager starting. + conf.setBoolean("hbase.master.start.wait.for.namespacemanager", true); conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java index f6f9b49..c273afa 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java @@ -66,6 +66,9 @@ public class TestMasterMetrics { public static void startCluster() throws Exception { LOG.info("Starting cluster"); TEST_UTIL = new HBaseTestingUtility(); + // The metrics depends on namespace region is online, therefore, we have to + // wait for namespace manager starting. + TEST_UTIL.getConfiguration().setBoolean("hbase.master.start.wait.for.namespacemanager", true); TEST_UTIL.startMiniCluster(1, 1, 1, null, MyMaster.class, null); cluster = TEST_UTIL.getHBaseCluster(); LOG.info("Waiting for active/ready master"); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java index 4483fce..3b1daf2 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java @@ -215,10 +215,6 @@ public class TestMasterNoCluster { public ClusterConnection getConnection() { return mockedConnection; } - - @Override - void initNamespace() { - } }; master.start(); @@ -296,10 +292,6 @@ public class TestMasterNoCluster { return null; } } - - @Override - void initNamespace() { - } }; master.start(); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java index f25e45c..eed85c1 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java @@ -54,10 +54,10 @@ public class TestCreateTableHandler { private static final Log LOG = LogFactory.getLog(TestCreateTableHandler.class); private static final byte[] FAMILYNAME = Bytes.toBytes("fam"); private static boolean throwException = false; - @Before public void setUp() throws Exception { + TEST_UTIL.getConfiguration().setBoolean("hbase.master.start.wait.for.namespacemanager", true); TEST_UTIL.startMiniCluster(1); } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateNamespaceProcedure.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateNamespaceProcedure.java index e7ddff6..c59b27d 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateNamespaceProcedure.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateNamespaceProcedure.java @@ -52,6 +52,10 @@ public class TestCreateNamespaceProcedure { private static void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + // We have to disable the async namespace manager startup due to the recovery and rollback + // tests would halt the execution of procedure. If at that time the namespace table is + // not created, it would be forced to stop/wait and cause the test to fail. + conf.setBoolean("hbase.master.start.wait.for.namespacemanager", true); } @BeforeClass diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureEvents.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureEvents.java index b7a6434..5540955 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureEvents.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureEvents.java @@ -76,14 +76,58 @@ public class TestMasterProcedureEvents { ProcedureExecutor procExec = master.getMasterProcedureExecutor(); MasterProcedureScheduler procSched = procExec.getEnvironment().getProcedureQueue(); - HRegionInfo hri = new HRegionInfo(tableName); HTableDescriptor htd = new HTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor("f"); htd.addFamily(hcd); while (!master.isInitialized()) Thread.sleep(250); + UTIL.createTable(htd, null); + + // Modify the table descriptor + HTableDescriptor newHTD = + new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName)); + long newMaxFileSize = newHTD.getMaxFileSize() * 2; + newHTD.setMaxFileSize(newMaxFileSize); + newHTD.setRegionReplication(3); + master.setInitialized(false); // fake it, set back later + ModifyTableProcedure proc = new ModifyTableProcedure(procExec.getEnvironment(), newHTD); + + long pollCalls = procSched.getPollCalls(); + long nullPollCalls = procSched.getNullPollCalls(); + + long procId = procExec.submitProcedure(proc); + for (int i = 0; i < 10; ++i) { + Thread.sleep(100); + assertEquals(pollCalls + 1, procSched.getPollCalls()); + assertEquals(nullPollCalls, procSched.getNullPollCalls()); + } + + master.setInitialized(true); + ProcedureTestingUtility.waitProcedure(procExec, procId); + + assertEquals(pollCalls + 2, procSched.getPollCalls()); + assertEquals(nullPollCalls, procSched.getNullPollCalls()); + } + + @Test + public void testNamespaceManagerInitializedEvent() throws Exception { + TableName tableName = TableName.valueOf("testNamespaceManagerInitializedEvent"); + HMaster master = UTIL.getMiniHBaseCluster().getMaster(); + ProcedureExecutor procExec = master.getMasterProcedureExecutor(); + MasterProcedureScheduler procSched = procExec.getEnvironment().getProcedureQueue(); + + HRegionInfo hri = new HRegionInfo(tableName); + HTableDescriptor htd = new HTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor("f"); + htd.addFamily(hcd); + + while (!master.isInitialized()) Thread.sleep(250); + UTIL.waitUntilTableNamespaceManagerStarted(); + + master.setNamespaceManagerInitializedEvent(false); // fake it, set back later + CreateTableProcedure proc = new CreateTableProcedure( procExec.getEnvironment(), htd, new HRegionInfo[] { hri }); @@ -97,7 +141,7 @@ public class TestMasterProcedureEvents { assertEquals(nullPollCalls, procSched.getNullPollCalls()); } - master.setInitialized(true); + master.setNamespaceManagerInitializedEvent(true); ProcedureTestingUtility.waitProcedure(procExec, procId); assertEquals(pollCalls + 2, procSched.getPollCalls()); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java index 327afc4..394b114 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java @@ -48,6 +48,10 @@ public class TestModifyNamespaceProcedure { private static void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + // We have to disable the async namespace manager startup due to the recovery and rollback + // tests would halt the execution of procedure. If at that time the namespace table is + // not created, it would be forced to stop/wait and cause the test to fail. + conf.setBoolean("hbase.master.start.wait.for.namespacemanager", true); } @BeforeClass diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java index aac872d..293a3c5 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java @@ -67,7 +67,9 @@ public class TestRegionOpen { public void testPriorityRegionIsOpenedWithSeparateThreadPool() throws Exception { ThreadPoolExecutor exec = getRS().getExecutorService() .getExecutorThreadPool(ExecutorType.RS_OPEN_PRIORITY_REGION); - + // This test depends on namespace region open; therefore, we have to wait for namespace + // manager start before continue. + HTU.waitUntilTableNamespaceManagerStarted(); assertEquals(1, exec.getCompletedTaskCount()); // namespace region HTableDescriptor htd = new HTableDescriptor(tableName);