diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchema.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchema.java index cb3b684..a1d662b 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchema.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchema.java @@ -37,7 +37,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; // coalescing common code. In particular, we'd contain filesystem modification. Other // benefits are to make all schema modification work the same way (one way to do an operation only // rather than the current approach where how an operation is done varies with context) and to make -// it so clusterschema modification can stand apart from Master to faciliate standalone +// it so clusterschema modification can stand apart from Master to facilitate standalone // testing. It is part of the filesystem refactor project that undoes the dependency on a // layout in HDFS that mimics our model of tables have regions have column families have files. // With this Interface in place, with all modifications going via this route where no filesystem diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaService.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaService.java index 43353ba..fae69aa 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaService.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaService.java @@ -24,4 +24,9 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; * Mixes in ClusterSchema and Service */ @InterfaceAudience.Private -public interface ClusterSchemaService extends ClusterSchema, Service {} \ No newline at end of file +public interface ClusterSchemaService extends ClusterSchema, Service { + /** + * @return True if this Service is fully started. + */ + boolean isStarted(); +} \ No newline at end of file diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaServiceImpl.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaServiceImpl.java index 0250f36..994a118 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaServiceImpl.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/ClusterSchemaServiceImpl.java @@ -23,6 +23,8 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.NamespaceNotFoundException; import org.apache.hadoop.hbase.ServiceNotRunningException; @@ -36,7 +38,9 @@ import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; @InterfaceAudience.Private class ClusterSchemaServiceImpl implements ClusterSchemaService { + private static final Log LOG = LogFactory.getLog(ClusterSchemaServiceImpl.class); private boolean running = false; + private boolean started = false; private final TableNamespaceManager tableNamespaceManager; private final MasterServices masterServices; private final static List EMPTY_NAMESPACE_LIST = @@ -54,16 +58,25 @@ class ClusterSchemaServiceImpl implements ClusterSchemaService { return this.running; } + @Override + public boolean isStarted() { + return this.started; + } + private synchronized void checkIsRunning() throws ServiceNotRunningException { if (!isRunning()) throw new ServiceNotRunningException(); } @Override public synchronized void startAndWait() throws IOException { - if (isRunning()) throw new IllegalStateException("Already running; cannot double-start."); + if (isRunning()) { + throw new IllegalStateException("Already running; cannot double-start."); + } // Set to running FIRST because tableNamespaceManager start uses this class to do namespace ops this.running = true; this.tableNamespaceManager.start(); + this.started = true; + LOG.info("Cluster Schema Service starts successfully."); } @Override @@ -71,6 +84,7 @@ class ClusterSchemaServiceImpl implements ClusterSchemaService { checkIsRunning(); // You can't stop tableNamespaceManager. this.running = false; + this.started = false; } @Override diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 0e07ae0..35aed5d 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -119,7 +119,6 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler.Procedu import org.apache.hadoop.hbase.master.procedure.ModifyColumnFamilyProcedure; import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure; import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch; -import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait; import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; import org.apache.hadoop.hbase.mob.MobConstants; @@ -164,7 +163,6 @@ import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker; import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker; import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; import org.apache.hadoop.hbase.zookeeper.MasterMaintenanceModeTracker; -import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker; import org.apache.hadoop.hbase.zookeeper.RegionServerTracker; import org.apache.hadoop.hbase.zookeeper.SplitOrMergeTracker; @@ -797,8 +795,8 @@ public class HMaster extends HRegionServer implements MasterServices { periodicDoMetricsChore = new PeriodicDoMetrics(msgInterval, this); getChoreService().scheduleChore(periodicDoMetricsChore); - status.setStatus("Starting cluster schema service"); - initClusterSchemaService(); + status.setStatus("Starting cluster schema service and quota manager asynchornizely"); + initClusterSchemaServiceAndQuotaManager(); if (this.cpHost != null) { try { @@ -818,9 +816,6 @@ public class HMaster extends HRegionServer implements MasterServices { status.setStatus("Assign meta replicas"); metaBootstrap.assignMetaReplicas(); - status.setStatus("Starting quota manager"); - initQuotaManager(); - // clear the dead servers with same host name and port of online server because we are not // removing dead server with same hostname and port of rs which is trying to check in before // master initialization. See HBASE-5916. @@ -894,17 +889,31 @@ public class HMaster extends HRegionServer implements MasterServices { } } - void initClusterSchemaService() throws IOException, InterruptedException { + void initClusterSchemaServiceAndQuotaManager() throws IOException, InterruptedException { this.clusterSchemaService = new ClusterSchemaServiceImpl(this); - this.clusterSchemaService.startAndWait(); - if (!this.clusterSchemaService.isRunning()) throw new HBaseIOException("Failed start"); - } - void initQuotaManager() throws IOException { - MasterQuotaManager quotaManager = new MasterQuotaManager(this); + this.quotaManager = new MasterQuotaManager(this); this.assignmentManager.setRegionStateListener((RegionStateListener)quotaManager); - quotaManager.start(); - this.quotaManager = quotaManager; + + Threads.setDaemonThreadRunning(new Thread(new Runnable() { + @Override + public void run() { + // Start cluster schema service and wait to it to be fully started. + try { + clusterSchemaService.startAndWait(); + } catch (IOException e) { + LOG.error("Cluster schema service failed to start. ", e); + return; + } + // Quota Manager depends on Cluster Schema Service to be fully initialized. + try { + quotaManager.start(); + } catch (IOException ie) { + LOG.warn("Quota Manager failed to start. Retry...", ie); + return; + } + } + }, "Init Cluster Schema Service and Quota Manager")); } boolean isCatalogJanitorEnabled() { @@ -1451,7 +1460,7 @@ public class HMaster extends HRegionServer implements MasterServices { if (isStopped()) { throw new MasterNotRunningException(); } - checkInitialized(); + checkClusterSchemaInitialized(); String namespace = hTableDescriptor.getTableName().getNamespaceAsString(); this.clusterSchemaService.getNamespace(namespace); @@ -2313,6 +2322,15 @@ public class HMaster extends HRegionServer implements MasterServices { if (!isInitialized()) throw new PleaseHoldException("Master is initializing"); } + void checkClusterSchemaInitialized() + throws PleaseHoldException, ServerNotRunningYetException { + checkInitialized(); + if (!isClusterSchemaServiceStarted()) { + throw new ServerNotRunningYetException( + "Cluster Schema Service is NOT fully initialized."); + } + } + /** * Report whether this master is currently the active master or not. * If not active master, we are parked on ZK waiting to become active. @@ -2341,6 +2359,16 @@ public class HMaster extends HRegionServer implements MasterServices { } /** + * Report whether cluster schema service is started. + * + * @return true if cluster schema service is started + */ + @Override + public boolean isClusterSchemaServiceStarted() { + return clusterSchemaService.isStarted(); + } + + /** * Report whether this master is in maintenance mode. * * @return true if master is in maintenanceMode @@ -2505,7 +2533,7 @@ public class HMaster extends HRegionServer implements MasterServices { long createNamespace(final NamespaceDescriptor namespaceDescriptor, final long nonceGroup, final long nonce) throws IOException { - checkInitialized(); + checkClusterSchemaInitialized(); TableName.isLegalNamespaceName(Bytes.toBytes(namespaceDescriptor.getName())); if (this.cpHost != null && this.cpHost.preCreateNamespace(namespaceDescriptor)) { throw new BypassCoprocessorException(); @@ -2527,7 +2555,7 @@ public class HMaster extends HRegionServer implements MasterServices { long modifyNamespace(final NamespaceDescriptor namespaceDescriptor, final long nonceGroup, final long nonce) throws IOException { - checkInitialized(); + checkClusterSchemaInitialized(); TableName.isLegalNamespaceName(Bytes.toBytes(namespaceDescriptor.getName())); if (this.cpHost != null && this.cpHost.preModifyNamespace(namespaceDescriptor)) { throw new BypassCoprocessorException(); @@ -2548,7 +2576,7 @@ public class HMaster extends HRegionServer implements MasterServices { */ long deleteNamespace(final String name, final long nonceGroup, final long nonce) throws IOException { - checkInitialized(); + checkClusterSchemaInitialized(); if (this.cpHost != null && this.cpHost.preDeleteNamespace(name)) { throw new BypassCoprocessorException(); } @@ -2565,9 +2593,9 @@ public class HMaster extends HRegionServer implements MasterServices { * @return Namespace descriptor for name */ NamespaceDescriptor getNamespace(String name) throws IOException { - checkInitialized(); + checkClusterSchemaInitialized(); if (this.cpHost != null) this.cpHost.preGetNamespaceDescriptor(name); - NamespaceDescriptor nsd = this.clusterSchemaService.getNamespace(name); + NamespaceDescriptor nsd = getClusterSchema().getNamespace(name); if (this.cpHost != null) this.cpHost.postGetNamespaceDescriptor(nsd); return nsd; } @@ -2577,14 +2605,14 @@ public class HMaster extends HRegionServer implements MasterServices { * @return All Namespace descriptors */ List getNamespaces() throws IOException { - checkInitialized(); + checkClusterSchemaInitialized(); final List nsds = new ArrayList(); boolean bypass = false; if (cpHost != null) { bypass = cpHost.preListNamespaceDescriptors(nsds); } if (!bypass) { - nsds.addAll(this.clusterSchemaService.getNamespaces()); + nsds.addAll(getClusterSchema().getNamespaces()); if (this.cpHost != null) this.cpHost.postListNamespaceDescriptors(nsds); } return nsds; @@ -2683,14 +2711,16 @@ public class HMaster extends HRegionServer implements MasterServices { */ private List getTableDescriptors(final List htds, final String namespace, final String regex, final List tableNameList, - final boolean includeSysTables) - throws IOException { + final boolean includeSysTables) throws IOException { if (tableNameList == null || tableNameList.size() == 0) { // request for all TableDescriptors Collection allHtds; if (namespace != null && namespace.length() > 0) { + // Make sure cluster schema service is initialized. + checkClusterSchemaInitialized(); + // Do a check on the namespace existence. Will fail if does not exist. - this.clusterSchemaService.getNamespace(namespace); + getClusterSchema().getNamespace(namespace); allHtds = tableDescriptors.getByNamespace(namespace).values(); } else { allHtds = tableDescriptors.getAll().values(); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 326aa00..b3c0b0a 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.ProcedureInfo; import org.apache.hadoop.hbase.ProcedureUtil; import org.apache.hadoop.hbase.ServerLoad; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.ServiceNotRunningException; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.UnknownRegionException; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -1158,7 +1159,7 @@ public class MasterRpcServices extends RSRpcServices public RestoreSnapshotResponse restoreSnapshot(RpcController controller, RestoreSnapshotRequest request) throws ServiceException { try { - master.checkInitialized(); + master.checkClusterSchemaInitialized(); master.snapshotManager.checkSnapshotSupport(); // Ensure namespace exists. Will throw exception if non-known NS. @@ -1341,7 +1342,11 @@ public class MasterRpcServices extends RSRpcServices public SetQuotaResponse setQuota(RpcController c, SetQuotaRequest req) throws ServiceException { try { - master.checkInitialized(); + master.checkClusterSchemaInitialized(); + if (!master.getMasterQuotaManager().isQuotaEnabled()) { + throw new ServiceNotRunningException( + "Quota manager service is NOT fully initialized. Please retry later..."); + } return master.getMasterQuotaManager().setQuota(req); } catch (Exception e) { throw new ServiceException(e); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java index 670642f..ce0ac80 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java @@ -317,6 +317,13 @@ public interface MasterServices extends Server { boolean isInitialized(); /** + * Report whether cluster schema service is started. + * + * @return true if cluster schema service is started + */ + boolean isClusterSchemaServiceStarted(); + + /** * @return true if master is in maintanceMode */ boolean isInMaintenanceMode(); diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java index cc01046..f2fed58 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java @@ -60,7 +60,7 @@ import com.google.common.collect.Sets; * {@link org.apache.hadoop.hbase.ZKNamespaceManager}. * * WARNING: Do not use. Go via the higher-level {@link ClusterSchema} API instead. This manager - * is likely to go aways anyways. + * is likely to go away anyways. */ @InterfaceAudience.Private @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="IS2_INCONSISTENT_SYNC", @@ -77,8 +77,6 @@ public class TableNamespaceManager { public static final String KEY_MAX_REGIONS = "hbase.namespace.quota.maxregions"; public static final String KEY_MAX_TABLES = "hbase.namespace.quota.maxtables"; - static final String NS_INIT_TIMEOUT = "hbase.master.namespace.init.timeout"; - static final int DEFAULT_NS_INIT_TIMEOUT = 300000; TableNamespaceManager(MasterServices masterServices) { this.masterServices = masterServices; @@ -94,13 +92,17 @@ public class TableNamespaceManager { try { // Wait for the namespace table to be initialized. - long startTime = EnvironmentEdgeManager.currentTime(); - int timeout = conf.getInt(NS_INIT_TIMEOUT, DEFAULT_NS_INIT_TIMEOUT); + final long startTime = EnvironmentEdgeManager.currentTime(); + final int intervalIncrement = 300000; + int totalWaitInterval = intervalIncrement; while (!isTableAvailableAndInitialized()) { - if (EnvironmentEdgeManager.currentTime() - startTime + 100 > timeout) { + if (EnvironmentEdgeManager.currentTime() - startTime + 100 > totalWaitInterval) { // We can't do anything if ns is not online. - throw new IOException("Timedout " + timeout + "ms waiting for namespace table to " - + "be assigned and enabled: " + getTableState()); + LOG.warn("Waiting " + totalWaitInterval + + "ms for namespace table to be assigned and enabled: " + getTableState() + + " Keep waiting..."); + //update the interval + totalWaitInterval += intervalIncrement; } Thread.sleep(100); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java index 1937528..98eeb7d 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java @@ -261,7 +261,9 @@ public class CreateNamespaceProcedure protected static void setNamespaceQuota( final MasterProcedureEnv env, final NamespaceDescriptor nsDescriptor) throws IOException { - if (env.getMasterServices().isInitialized()) { + if (env.getMasterServices().isInitialized() && + env.getMasterServices().isClusterSchemaServiceStarted() && + env.getMasterServices().getMasterQuotaManager().isQuotaEnabled()) { env.getMasterServices().getMasterQuotaManager().setNamespaceQuota(nsDescriptor); } } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java index 8333919..62fdbab 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/DeleteNamespaceProcedure.java @@ -214,6 +214,9 @@ public class DeleteNamespaceProcedure * @throws IOException */ private void prepareDelete(final MasterProcedureEnv env) throws IOException { + if (!env.getMasterServices().isClusterSchemaServiceStarted()) { + throw new IOException("Cluster Schema Servicen is NOT started. Cannot do namespace DDL"); + } if (getTableNamespaceManager(env).doesNamespaceExist(namespaceName) == false) { throw new NamespaceNotFoundException(namespaceName); } @@ -360,9 +363,15 @@ public class DeleteNamespaceProcedure } } + /** + * The procedure could be restarted from a different machine. If the variable is null, we need to + * retrieve it. + * @return tableNamespaceManager + */ private static TableNamespaceManager getTableNamespaceManager(final MasterProcedureEnv env) { - return env.getMasterServices().getClusterSchema().getTableNamespaceManager(); + return MasterDDLOperationHelper.getInitializedTableNamespaceManager(env); } + /** * The procedure could be restarted from a different machine. If the variable is null, we need to * retrieve it. diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java index 1214268..e7b5e03 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterDDLOperationHelper.java @@ -32,16 +32,16 @@ import org.apache.hadoop.hbase.HRegionLocation; import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.RegionLocator; -import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.master.AssignmentManager; import org.apache.hadoop.hbase.master.BulkReOpen; import org.apache.hadoop.hbase.master.MasterFileSystem; +import org.apache.hadoop.hbase.master.TableNamespaceManager; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -163,4 +163,16 @@ public final class MasterDDLOperationHelper { } return regionInfoList; } + + /** + * Get the TableNamespaceManager for namespace-related DDL operation. + **/ + public static TableNamespaceManager getInitializedTableNamespaceManager( + final MasterProcedureEnv env) { + while (!env.getMasterServices().isClusterSchemaServiceStarted()) { + // Sleep some + Threads.sleep(100); + } + return env.getMasterServices().getClusterSchema().getTableNamespaceManager(); + } } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java index 197290c..a2572be 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyNamespaceProcedure.java @@ -45,10 +45,12 @@ public class ModifyNamespaceProcedure private NamespaceDescriptor oldNsDescriptor; private NamespaceDescriptor newNsDescriptor; + private TableNamespaceManager nsManager; private Boolean traceEnabled; public ModifyNamespaceProcedure() { this.oldNsDescriptor = null; + this.nsManager = null; this.traceEnabled = null; } @@ -56,6 +58,7 @@ public class ModifyNamespaceProcedure final NamespaceDescriptor newNsDescriptor) { this.oldNsDescriptor = null; this.newNsDescriptor = newNsDescriptor; + this.nsManager = null; this.traceEnabled = null; this.setOwner(env.getRequestUser().getUGI().getShortUserName()); } @@ -199,6 +202,10 @@ public class ModifyNamespaceProcedure * @throws IOException */ private void prepareModify(final MasterProcedureEnv env) throws IOException { + if (!env.getMasterServices().isClusterSchemaServiceStarted()) { + throw new IOException("Cluster Schema Servicen is NOT started. Cannot do namespace DDL"); + } + if (getTableNamespaceManager(env).doesNamespaceExist(newNsDescriptor.getName()) == false) { throw new NamespaceNotFoundException(newNsDescriptor.getName()); } @@ -226,8 +233,16 @@ public class ModifyNamespaceProcedure getTableNamespaceManager(env).updateZKNamespaceManager(newNsDescriptor); } + /** + * The procedure could be restarted from a different machine. If the variable is null, we need to + * retrieve it. + * @return tableNamespaceManager + */ private TableNamespaceManager getTableNamespaceManager(final MasterProcedureEnv env) { - return env.getMasterServices().getClusterSchema().getTableNamespaceManager(); + if (this.nsManager == null) { + this.nsManager = MasterDDLOperationHelper.getInitializedTableNamespaceManager(env); + } + return this.nsManager; } /** diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java index 79865bb..b94c6c2 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/JVMClusterUtil.java @@ -212,7 +212,7 @@ public class JVMClusterUtil { final int maxwait = 200000; while (true) { JVMClusterUtil.MasterThread t = findActiveMaster(masters); - if (t != null && t.master.isInitialized()) { + if (t != null && t.master.isInitialized() && t.master.isClusterSchemaServiceStarted()) { return t.master.getServerName().toString(); } // REMOVE diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java index 706d8e7..7775de2 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java @@ -529,7 +529,10 @@ public class MiniHBaseCluster extends HBaseCluster { while (!(mts = getMasterThreads()).isEmpty() && (System.currentTimeMillis() - start) < timeout) { for (JVMClusterUtil.MasterThread mt : mts) { - if (mt.getMaster().isActiveMaster() && mt.getMaster().isInitialized()) { + HMaster master = mt.getMaster(); + if (master.isActiveMaster() && + master.isInitialized() && + master.isClusterSchemaServiceStarted()) { return true; } } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java index 56a8522..091158e 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java @@ -312,6 +312,11 @@ public class MockNoopMasterServices implements MasterServices, Server { } @Override + public boolean isClusterSchemaServiceStarted() { + return false; + } + + @Override public boolean isInMaintenanceMode() { return false; } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java index 6da7a38..40df6f3 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java @@ -204,9 +204,6 @@ public class TestMasterNoCluster { } @Override - void initClusterSchemaService() throws IOException, InterruptedException {} - - @Override ServerManager createServerManager(MasterServices master) throws IOException { ServerManager sm = super.createServerManager(master); // Spy on the created servermanager @@ -275,9 +272,6 @@ public class TestMasterNoCluster { } @Override - void initClusterSchemaService() throws IOException, InterruptedException {} - - @Override void initializeZKBasedSystemTrackers() throws IOException, InterruptedException, KeeperException, CoordinatedStateException { super.initializeZKBasedSystemTrackers();