From f4561617c2ea7d0c3b0801578ed43dbd65323d23 Mon Sep 17 00:00:00 2001 From: Xu Cang Date: Tue, 23 Apr 2019 13:16:09 -0700 Subject: [PATCH] HBASE-16488 Starting namespace and quota services in master --- .../apache/hadoop/hbase/master/HMaster.java | 105 +++++++++++++++--- .../hbase/master/MasterRpcServices.java | 4 +- .../hbase/master/TableNamespaceManager.java | 21 +++- .../procedure/CreateNamespaceProcedure.java | 19 ++-- .../procedure/CreateTableProcedure.java | 2 +- .../procedure/DeleteNamespaceProcedure.java | 4 +- .../master/procedure/MasterProcedureEnv.java | 4 + .../procedure/ModifyNamespaceProcedure.java | 4 +- .../hadoop/hbase/HBaseTestingUtility.java | 34 ++++++ .../hbase/client/TestRollbackFromClient.java | 2 +- .../hbase/master/TestAssignmentListener.java | 22 +++- .../hbase/master/TestMasterFailover.java | 3 + .../hbase/master/TestMasterMetrics.java | 3 + .../hbase/master/TestMasterNoCluster.java | 8 -- .../handler/TestCreateTableHandler.java | 2 +- .../TestCreateNamespaceProcedure.java | 4 + .../procedure/TestMasterProcedureEvents.java | 71 +++++++++++- .../TestModifyNamespaceProcedure.java | 4 + .../hbase/regionserver/TestRegionOpen.java | 7 ++ .../wal/TestWALOpenAfterDNRollingStart.java | 1 + 20 files changed, 273 insertions(+), 51 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index e8b9a8193d..fa9906c2b0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -160,6 +160,7 @@ import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.CompressionTest; import org.apache.hadoop.hbase.util.ConfigUtil; import org.apache.hadoop.hbase.util.EncryptionTest; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.util.HasThread; import org.apache.hadoop.hbase.util.ModifyRegionUtils; @@ -313,6 +314,10 @@ public class HMaster extends HRegionServer implements MasterServices, Server { // initialization may have not completed yet. volatile boolean serviceStarted = false; + // flag set after we complete asynchronized services and master initialization is done, + private final ProcedureEvent namespaceManagerInitialized = + new ProcedureEvent("master namespace manager initialized"); + // flag set after we complete assignMeta. private final ProcedureEvent serverCrashProcessingEnabled = new ProcedureEvent("server crash processing"); @@ -880,8 +885,9 @@ public class HMaster extends HRegionServer implements MasterServices, Server { periodicDoMetricsChore = new PeriodicDoMetrics(msgInterval, this); getChoreService().scheduleChore(periodicDoMetricsChore); - status.setStatus("Starting namespace manager"); - initNamespace(); + + status.setStatus("Starting namespace manager and quota manager"); + initNamespaceAndQuotaManager(); if (this.cpHost != null) { try { @@ -902,11 +908,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server { // Set master as 'initialized'. setInitialized(true); - assignmentManager.checkIfShouldMoveSystemRegionAsync(); - - status.setStatus("Starting quota manager"); - initQuotaManager(); - // assign the meta replicas Set EMPTY_SET = new HashSet(); int numReplicas = conf.getInt(HConstants.META_REPLICAS_NUM, @@ -949,12 +950,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } } - private void initQuotaManager() throws IOException { - quotaManager = new MasterQuotaManager(this); - this.assignmentManager.setRegionStateListener((RegionStateListener) quotaManager); - quotaManager.start(); - } - /** * Create a {@link ServerManager} instance. * @param master @@ -1101,12 +1096,61 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } } - void initNamespace() throws IOException { + /* + * The main purpose is to start namespace manager and quota manager async to + * unblock overall master initialization + * + * @throws IOException + */ + private void initNamespaceAndQuotaManager() throws IOException { //create namespace manager tableNamespaceManager = new TableNamespaceManager(this); - tableNamespaceManager.start(); + //create quota manager + this.quotaManager = new MasterQuotaManager(this); + this.assignmentManager.setRegionStateListener((RegionStateListener)quotaManager); + + if (conf.getBoolean("hbase.master.start.wait.for.namespacemanager", false)) { + // If being asked not to async start namespace manager, then just block + // master service starting until namespace manager is ready. + // + // Note: Quota manager depends on namespace manager. Therefore, its starting + // method has to be in-sync with namespace manager. + LOG.info("Starting namespace manager and quota manager synchronously"); + + tableNamespaceManager.start(); + setNamespaceManagerInitializedEvent(true); + + quotaManager.start(); + } else { // else asynchronously start namespace manager and quota manager + LOG.info("Starting namespace manager and quota manager asynchronously"); + Threads.setDaemonThreadRunning(new Thread(new Runnable() { + @Override + public void run() { + // Start namespace manager and wait to it to be fully started. + try { + tableNamespaceManager.start(); + setNamespaceManagerInitializedEvent(true); + LOG.info("Namespace manager started."); + } catch (IOException e) { + LOG.error("Namespace manager failed to start. ", e); + abort("Shutdown Master due to namespace manager failed to start. ", e); + } + // Quota Manager depends on Namespace manager to be fully initialized. + try { + quotaManager.start(); + LOG.info("Quota manager started."); + } catch (IOException ie) { + LOG.error("Quota Manager failed to start. ", ie); + abort("Shutdown Master due to Quota Manager failure to start. ", ie); + } + } + }, "Init Namespace Manager and Quota Manager Async")); + } + assignmentManager.checkIfShouldMoveSystemRegionAsync(); } + + boolean isCatalogJanitorEnabled() { return catalogJanitorChore != null ? catalogJanitorChore.getEnabled() : false; @@ -2665,11 +2709,28 @@ public class HMaster extends HRegionServer implements MasterServices, Server { void checkNamespaceManagerReady() throws IOException { checkInitialized(); - if (tableNamespaceManager == null || - !tableNamespaceManager.isTableAvailableAndInitialized(true)) { + + if (tableNamespaceManager == null) { throw new IOException("Table Namespace Manager not ready yet, try again later"); + } else if (!tableNamespaceManager.isTableAvailableAndInitialized(true)) { + try { + // Wait some time. + long startTime = EnvironmentEdgeManager.currentTime(); + int timeout = conf.getInt("hbase.master.namespace.wait.for.ready", 30000); + while (!tableNamespaceManager.isTableNamespaceManagerStarted() && + EnvironmentEdgeManager.currentTime() - startTime < timeout) { + Thread.sleep(100); + } + } catch (InterruptedException e) { + throw (InterruptedIOException) new InterruptedIOException().initCause(e); + } + if (!tableNamespaceManager.isTableNamespaceManagerStarted()) { + throw new IOException("Table Namespace Manager not fully initialized, try again later"); + } } } + + /** * Report whether this master is currently the active master or not. * If not active master, we are parked on ZK waiting to become active. @@ -2718,6 +2779,16 @@ public class HMaster extends HRegionServer implements MasterServices, Server { return initialized; } + public void setNamespaceManagerInitializedEvent(boolean isNamespaceManagerInitialized) { + procedureExecutor.getEnvironment().setEventReady( + namespaceManagerInitialized, + isNamespaceManagerInitialized); + } + + public ProcedureEvent getNamespaceManagerInitializedEvent() { + return namespaceManagerInitialized; + } + /** * ServerCrashProcessingEnabled is set false before completing assignMeta to prevent processing * of crashed servers. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 6b674732ae..ee6f6f3d0b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -1323,7 +1323,7 @@ public class MasterRpcServices extends RSRpcServices public RestoreSnapshotResponse restoreSnapshot(RpcController controller, RestoreSnapshotRequest request) throws ServiceException { try { - master.checkInitialized(); + master.checkNamespaceManagerReady(); master.snapshotManager.checkSnapshotSupport(); // ensure namespace exists @@ -1631,7 +1631,7 @@ public class MasterRpcServices extends RSRpcServices @Override public SetQuotaResponse setQuota(RpcController c, SetQuotaRequest req) throws ServiceException { try { - master.checkInitialized(); + master.checkNamespaceManagerReady(); return master.getMasterQuotaManager().setQuota(req); } catch (Exception e) { throw new ServiceException(e); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java index 6bcfb7719b..33a2fcf426 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java @@ -18,6 +18,9 @@ package org.apache.hadoop.hbase.master; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Sets; + import java.io.IOException; import java.io.InterruptedIOException; import java.util.NavigableSet; @@ -48,8 +51,6 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import com.google.common.collect.Sets; - /** * This is a helper class used to manage the namespace * metadata that is stored in TableName.NAMESPACE_TABLE_NAME @@ -72,7 +73,8 @@ public class TableNamespaceManager { public static final String KEY_MAX_REGIONS = "hbase.namespace.quota.maxregions"; public static final String KEY_MAX_TABLES = "hbase.namespace.quota.maxtables"; static final String NS_INIT_TIMEOUT = "hbase.master.namespace.init.timeout"; - static final int DEFAULT_NS_INIT_TIMEOUT = 300000; + static final int DEFAULT_NS_INIT_TIMEOUT = 1800000; // default is 30 minutes + private static final int WAIT_MESSAGE_TO_PRINTOUT = 300000; // print out message every 5 minutes public TableNamespaceManager(MasterServices masterServices) { this.masterServices = masterServices; @@ -91,12 +93,18 @@ public class TableNamespaceManager { // If timed out, we will move ahead without initializing it. // So that it should be initialized later on lazily. long startTime = EnvironmentEdgeManager.currentTime(); + long msgCount = 0; + long waitTime; int timeout = conf.getInt(NS_INIT_TIMEOUT, DEFAULT_NS_INIT_TIMEOUT); while (!isTableAvailableAndInitialized(false)) { - if (EnvironmentEdgeManager.currentTime() - startTime + 100 > timeout) { + waitTime = EnvironmentEdgeManager.currentTime() - startTime; + if (waitTime > timeout) { // We can't do anything if ns is not online. throw new IOException("Timedout " + timeout + "ms waiting for namespace table to " + "be assigned"); + } else if (waitTime > msgCount * WAIT_MESSAGE_TO_PRINTOUT) { + LOG.info("Waiting for namespace table to be online. Time waited = " + waitTime + " ms."); + msgCount++; } Thread.sleep(100); } @@ -105,6 +113,11 @@ public class TableNamespaceManager { } } + @VisibleForTesting + public boolean isTableNamespaceManagerStarted() { + return initialized; + } + private synchronized Table getNamespaceTable() throws IOException { if (!isTableNamespaceManagerInitialized()) { throw new IOException(this.getClass().getName() + " isn't ready to serve"); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java index e87315644e..d19635c14c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java @@ -193,21 +193,20 @@ public class CreateNamespaceProcedure sb.append(")"); } - private boolean isBootstrapNamespace() { + private static boolean isBootstrapNamespace(final NamespaceDescriptor nsDescriptor) { return nsDescriptor.equals(NamespaceDescriptor.DEFAULT_NAMESPACE) || nsDescriptor.equals(NamespaceDescriptor.SYSTEM_NAMESPACE); } @Override protected boolean acquireLock(final MasterProcedureEnv env) { - if (!env.getMasterServices().isInitialized()) { - // Namespace manager might not be ready if master is not fully initialized, - // return false to reject user namespace creation; return true for default - // and system namespace creation (this is part of master initialization). - if (!isBootstrapNamespace() && env.waitInitialized(this)) { - return false; - } + // Namespace manager might not be ready if master is not fully initialized, + // return false to reject user namespace creation; return true for default + // and system namespace creation (this is part of master initialization). + if (!isBootstrapNamespace(nsDescriptor) && env.waitNamespaceManagerInitialized(this)) { + return false; } + return env.getProcedureQueue().tryAcquireNamespaceExclusiveLock(this, getNamespaceName()); } @@ -331,7 +330,7 @@ public class CreateNamespaceProcedure protected static void setNamespaceQuota( final MasterProcedureEnv env, final NamespaceDescriptor nsDescriptor) throws IOException { - if (env.getMasterServices().isInitialized()) { + if (env.getMasterServices().isInitialized() && !isBootstrapNamespace(nsDescriptor)) { env.getMasterServices().getMasterQuotaManager().setNamespaceQuota(nsDescriptor); } } @@ -370,6 +369,6 @@ public class CreateNamespaceProcedure protected boolean shouldWaitClientAck(MasterProcedureEnv env) { // hbase and default namespaces are created on bootstrap internally by the system // the client does not know about this procedures. - return !isBootstrapNamespace(); + return !isBootstrapNamespace(nsDescriptor); } } \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java index 152af450a7..6de55fa4aa 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java @@ -275,7 +275,7 @@ public class CreateTableProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { - if (!getTableName().isSystemTable() && env.waitInitialized(this)) { + if (!getTableName().isSystemTable() && env.waitNamespaceManagerInitialized(this)) { return false; } return env.getProcedureQueue().tryAcquireTableExclusiveLock(this, getTableName()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java index a7ebc3063e..e011f29318 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java @@ -210,7 +210,9 @@ public class DeleteNamespaceProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { - if (env.waitInitialized(this)) return false; + if (env.waitNamespaceManagerInitialized(this)) { + return false; + } return env.getProcedureQueue().tryAcquireNamespaceExclusiveLock(this, getNamespaceName()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java index 3911f54790..75822c832c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureEnv.java @@ -132,6 +132,10 @@ public class MasterProcedureEnv { return procSched.waitEvent(((HMaster)master).getInitializedEvent(), proc); } + public boolean waitNamespaceManagerInitialized(Procedure proc) { + return procSched.waitEvent(((HMaster)master).getNamespaceManagerInitializedEvent(), proc); + } + public boolean waitServerCrashProcessingEnabled(Procedure proc) { return procSched.waitEvent(((HMaster)master).getServerCrashProcessingEnabledEvent(), proc); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java index 5d9548bdd6..5953032ebe 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java @@ -191,7 +191,9 @@ public class ModifyNamespaceProcedure @Override protected boolean acquireLock(final MasterProcedureEnv env) { - if (env.waitInitialized(this)) return false; + if (env.waitNamespaceManagerInitialized(this)) { + return false; + } return env.getProcedureQueue().tryAcquireNamespaceExclusiveLock(this, getNamespaceName()); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index cdebddbda1..c0092be7c9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -106,6 +106,7 @@ import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.visibility.VisibilityLabelsCache; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CommonFSUtils; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.JVMClusterUtil; @@ -1483,6 +1484,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { .setMaxVersions(numVersions); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc, startKey, endKey, numRegions); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1542,6 +1544,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { hcd.setBlocksize(blockSize); htd.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(htd, splitKeys); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are // assigned @@ -1558,6 +1561,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { */ public HTable createTable(HTableDescriptor htd, byte[][] splitRows) throws IOException { + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(htd, splitRows); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(htd.getTableName()); @@ -1612,6 +1616,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { hcd.setBloomFilterType(BloomType.NONE); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc); return new HTable(c, desc.getTableName()); } @@ -1634,6 +1639,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { .setMaxVersions(numVersions); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1658,6 +1664,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { .setMaxVersions(numVersions); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc); return new HTable(c, desc.getTableName()); } @@ -1732,6 +1739,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { HColumnDescriptor hcd = new HColumnDescriptor(family).setMaxVersions(numVersions); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc, splitKeys); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1784,6 +1792,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { .setBlocksize(blockSize); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1823,6 +1832,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { desc.addFamily(hcd); i++; } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1855,6 +1865,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor(family); desc.addFamily(hcd); + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc, splitRows); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(tableName); @@ -1887,12 +1898,35 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { HColumnDescriptor hcd = new HColumnDescriptor(family); desc.addFamily(hcd); } + waitUntilTableNamespaceManagerStarted(); getHBaseAdmin().createTable(desc, splitRows); // HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned waitUntilAllRegionsAssigned(desc.getTableName()); return new HTable(getConfiguration(), desc.getTableName()); } + public void waitUntilTableNamespaceManagerStarted() throws IOException { + final int maxwait = 60000; + final long startTime = EnvironmentEdgeManager.currentTime(); + final HMaster master = getMiniHBaseCluster().getMaster(); + do { + if(master.getTableNamespaceManager().isTableNamespaceManagerStarted()) { + break; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new IOException("Interrupt while waiting for master namespace manager starting."); + } + } while (EnvironmentEdgeManager.currentTime() - startTime < maxwait); + if(!master.getTableNamespaceManager().isTableNamespaceManagerStarted()) { + throw new IOException( + "Cannot continue testing due to master namespace manager not started after waiting " + + (EnvironmentEdgeManager.currentTime() - startTime) + " milliseconds"); + } + } + + /** * Create an unmanaged WAL. Be sure to close it when you're through. */ diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRollbackFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRollbackFromClient.java index f91adf4265..fc20d70650 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRollbackFromClient.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestRollbackFromClient.java @@ -222,7 +222,7 @@ public class TestRollbackFromClient { HColumnDescriptor col = new HColumnDescriptor(FAMILY); col.setMaxVersions(versions); desc.addFamily(col); - TEST_UTIL.getHBaseAdmin().createTable(desc); + TEST_UTIL.createTable(tableName, FAMILY, versions); int expected; List cells; try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java index 12d04ff8ca..bd5e0fd55f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentListener.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.concurrent.TimeUnit; @@ -84,6 +85,18 @@ public class TestAssignmentListener { Thread.sleep(100); } } + + //wait on multiple possible values. + public int awaitModifications(List values) throws InterruptedException { + while (true) { + for (int v : values) { + if (modified.compareAndSet(v, 0)) { + return v; + } + Thread.sleep(100); + } + } + } } static class DummyAssignmentListener extends DummyListener implements AssignmentListener { @@ -226,10 +239,13 @@ public class TestAssignmentListener { final byte[] FAMILY = Bytes.toBytes("cf"); // Create a new table, with a single region - LOG.info("Create Table"); + LOG.info("Create Table testtb"); TEST_UTIL.createTable(TABLE_NAME, FAMILY); - listener.awaitModifications(1); - assertEquals(1, listener.getLoadCount()); + // Wait for 1 or 2 since if namespace table is created before testtb, modified value will be 2 + // Waiting on 1 will fail this test indefinitely. + List modificationValues = Arrays.asList(1,2); + int modification = listener.awaitModifications(modificationValues); + assertEquals(modification, listener.getLoadCount()); assertEquals(0, listener.getCloseCount()); // Add some data diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java index 2228188357..47577c4626 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java @@ -508,6 +508,9 @@ public class TestMasterFailover { HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); Configuration conf = TEST_UTIL.getConfiguration(); conf.setBoolean("hbase.assignment.usezk", true); + // The test depends on namespace region is online, therefore, we have to + // wait for namespace manager starting. + conf.setBoolean("hbase.master.start.wait.for.namespacemanager", true); conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, 1); conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, 2); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java index f6f9b49860..c273afa2bb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterMetrics.java @@ -66,6 +66,9 @@ public class TestMasterMetrics { public static void startCluster() throws Exception { LOG.info("Starting cluster"); TEST_UTIL = new HBaseTestingUtility(); + // The metrics depends on namespace region is online, therefore, we have to + // wait for namespace manager starting. + TEST_UTIL.getConfiguration().setBoolean("hbase.master.start.wait.for.namespacemanager", true); TEST_UTIL.startMiniCluster(1, 1, 1, null, MyMaster.class, null); cluster = TEST_UTIL.getHBaseCluster(); LOG.info("Waiting for active/ready master"); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java index 4483fceaeb..3b1daf2ff3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java @@ -215,10 +215,6 @@ public class TestMasterNoCluster { public ClusterConnection getConnection() { return mockedConnection; } - - @Override - void initNamespace() { - } }; master.start(); @@ -296,10 +292,6 @@ public class TestMasterNoCluster { return null; } } - - @Override - void initNamespace() { - } }; master.start(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java index f25e45c129..eed85c1f6b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/handler/TestCreateTableHandler.java @@ -54,10 +54,10 @@ public class TestCreateTableHandler { private static final Log LOG = LogFactory.getLog(TestCreateTableHandler.class); private static final byte[] FAMILYNAME = Bytes.toBytes("fam"); private static boolean throwException = false; - @Before public void setUp() throws Exception { + TEST_UTIL.getConfiguration().setBoolean("hbase.master.start.wait.for.namespacemanager", true); TEST_UTIL.startMiniCluster(1); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateNamespaceProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateNamespaceProcedure.java index e7ddff6a7a..c59b27df3d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateNamespaceProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestCreateNamespaceProcedure.java @@ -52,6 +52,10 @@ public class TestCreateNamespaceProcedure { private static void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + // We have to disable the async namespace manager startup due to the recovery and rollback + // tests would halt the execution of procedure. If at that time the namespace table is + // not created, it would be forced to stop/wait and cause the test to fail. + conf.setBoolean("hbase.master.start.wait.for.namespacemanager", true); } @BeforeClass diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureEvents.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureEvents.java index b7a643423c..d23ab8a63a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureEvents.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestMasterProcedureEvents.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -43,6 +44,8 @@ import org.junit.experimental.categories.Category; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.io.IOException; + @Category({MasterTests.class, MediumTests.class}) public class TestMasterProcedureEvents { private static final Log LOG = LogFactory.getLog(TestCreateTableProcedure.class); @@ -76,13 +79,77 @@ public class TestMasterProcedureEvents { ProcedureExecutor procExec = master.getMasterProcedureExecutor(); MasterProcedureScheduler procSched = procExec.getEnvironment().getProcedureQueue(); + HTableDescriptor htd = new HTableDescriptor(tableName); + HColumnDescriptor hcd = new HColumnDescriptor("f"); + htd.addFamily(hcd); + + while (!master.isInitialized()) { + Thread.sleep(250); + } + UTIL.createTable(htd, null); + + // Modify the table descriptor + HTableDescriptor newHTD = + new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName)); + long newMaxFileSize = newHTD.getMaxFileSize() * 2; + newHTD.setMaxFileSize(newMaxFileSize); + newHTD.setRegionReplication(3); + + master.setInitialized(false); // fake it, set back later + + ModifyTableProcedure proc = new ModifyTableProcedure(procExec.getEnvironment(), newHTD); + + long pollCalls = procSched.getPollCalls(); + long nullPollCalls = procSched.getNullPollCalls(); + + long procId = procExec.submitProcedure(proc); + for (int i = 0; i < 10; ++i) { + Thread.sleep(100); + assertEquals(pollCalls + 1, procSched.getPollCalls()); + assertEquals(nullPollCalls, procSched.getNullPollCalls()); + } + + master.setInitialized(true); + ProcedureTestingUtility.waitProcedure(procExec, procId); + + assertEquals(pollCalls + 2, procSched.getPollCalls()); + assertEquals(nullPollCalls, procSched.getNullPollCalls()); + } + + @Test + public void testNamespaceManagerInitializedEvent() throws Exception { + TableName tableName = TableName.valueOf("testNamespaceManagerInitializedEvent"); + HMaster master = UTIL.getMiniHBaseCluster().getMaster(); + ProcedureExecutor procExec = master.getMasterProcedureExecutor(); + MasterProcedureScheduler procSched = procExec.getEnvironment().getProcedureQueue(); + HRegionInfo hri = new HRegionInfo(tableName); HTableDescriptor htd = new HTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor("f"); htd.addFamily(hcd); while (!master.isInitialized()) Thread.sleep(250); - master.setInitialized(false); // fake it, set back later + + final int maxwait = 60000; + final long startTime = EnvironmentEdgeManager.currentTime(); + do { + if(master.getTableNamespaceManager().isTableNamespaceManagerStarted()) { + break; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new IOException("Interrupt while waiting for master namespace manager starting."); + } + } while (EnvironmentEdgeManager.currentTime() - startTime < maxwait); + + if(!master.getTableNamespaceManager().isTableNamespaceManagerStarted()) { + throw new IOException( + "Cannot continue testing due to master namespace manager not started after waiting " + + (EnvironmentEdgeManager.currentTime() - startTime) + " milliseconds"); + } + + master.setNamespaceManagerInitializedEvent(false); // fake it, set back later CreateTableProcedure proc = new CreateTableProcedure( procExec.getEnvironment(), htd, new HRegionInfo[] { hri }); @@ -97,7 +164,7 @@ public class TestMasterProcedureEvents { assertEquals(nullPollCalls, procSched.getNullPollCalls()); } - master.setInitialized(true); + master.setNamespaceManagerInitializedEvent(true); ProcedureTestingUtility.waitProcedure(procExec, procId); assertEquals(pollCalls + 2, procSched.getPollCalls()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java index 327afc4f29..394b1147d9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestModifyNamespaceProcedure.java @@ -48,6 +48,10 @@ public class TestModifyNamespaceProcedure { private static void setupConf(Configuration conf) { conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + // We have to disable the async namespace manager startup due to the recovery and rollback + // tests would halt the execution of procedure. If at that time the namespace table is + // not created, it would be forced to stop/wait and cause the test to fail. + conf.setBoolean("hbase.master.start.wait.for.namespacemanager", true); } @BeforeClass diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java index f45df18f1c..e69d7170b6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java @@ -66,6 +66,9 @@ public class TestRegionOpen { @BeforeClass public static void before() throws Exception { + // This test depends on namespace region open; therefore, we have to wait for namespace + // manager start before continue. + HTU.getConfiguration().setBoolean("hbase.master.start.wait.for.namespacemanager", true); HTU.startMiniCluster(NB_SERVERS); } @@ -84,6 +87,10 @@ public class TestRegionOpen { ThreadPoolExecutor exec = getRS().getExecutorService() .getExecutorThreadPool(ExecutorType.RS_OPEN_PRIORITY_REGION); + // This test depends on namespace region open; therefore, we have to wait for namespace + // manager start before continue. + HTU.waitUntilTableNamespaceManagerStarted(); + assertEquals(1, exec.getCompletedTaskCount()); // namespace region HTableDescriptor htd = new HTableDescriptor(tableName); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALOpenAfterDNRollingStart.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALOpenAfterDNRollingStart.java index ee1692ea2b..4eb949fa6a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALOpenAfterDNRollingStart.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALOpenAfterDNRollingStart.java @@ -48,6 +48,7 @@ public class TestWALOpenAfterDNRollingStart { false); TEST_UTIL.getConfiguration().setLong("hbase.regionserver.hlog.check.lowreplication.interval", checkLowReplicationInterval); + TEST_UTIL.getConfiguration().setBoolean("hbase.master.start.wait.for.namespacemanager", true); TEST_UTIL.startMiniDFSCluster(3); TEST_UTIL.startMiniCluster(1); -- 2.17.2 (Apple Git-113)