From 6a9428b646aa0b8a198857ece1e2502f00f8f14b Mon Sep 17 00:00:00 2001 From: zhangduo Date: Thu, 12 Mar 2015 10:37:36 +0800 Subject: [PATCH] HBASE-13194 TableNamespaceManager not ready cause MasterQuotaManager initialization fail --- .../org/apache/hadoop/hbase/master/HMaster.java | 6 +++-- .../hadoop/hbase/master/TableNamespaceManager.java | 17 +++++------- .../hadoop/hbase/namespace/NamespaceAuditor.java | 15 ----------- .../hbase/namespace/NamespaceStateManager.java | 31 +++++++++------------- .../hbase/namespace/TestNamespaceAuditor.java | 19 +++++++------ 5 files changed, 32 insertions(+), 56 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 5f14aa9..0d6f35a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -288,7 +288,8 @@ public class HMaster extends HRegionServer implements MasterServices, Server { // monitor for distributed procedures MasterProcedureManagerHost mpmHost; - private MasterQuotaManager quotaManager; + // it is assigned after 'initialized' guard set to true, so should be volatile + private volatile MasterQuotaManager quotaManager; // handle table states private TableStateManager tableStateManager; @@ -897,9 +898,10 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } void initQuotaManager() throws IOException { - quotaManager = new MasterQuotaManager(this); + MasterQuotaManager quotaManager = new MasterQuotaManager(this); this.assignmentManager.setRegionStateListener((RegionStateListener)quotaManager); quotaManager.start(); + this.quotaManager = quotaManager; } boolean isCatalogJanitorEnabled() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java index 323ccee..f0f8fdd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java @@ -86,31 +86,26 @@ public class TableNamespaceManager { public void start() throws IOException { if (!MetaTableAccessor.tableExists(masterServices.getConnection(), - TableName.NAMESPACE_TABLE_NAME)) { + TableName.NAMESPACE_TABLE_NAME)) { LOG.info("Namespace table not found. Creating..."); createNamespaceTable(masterServices); } try { - // Wait for the namespace table to be assigned. - // If timed out, we will move ahead without initializing it. - // So that it should be initialized later on lazily. + // Wait for the namespace table to be initialized. long startTime = EnvironmentEdgeManager.currentTime(); int timeout = conf.getInt(NS_INIT_TIMEOUT, DEFAULT_NS_INIT_TIMEOUT); - while (!(isTableAssigned() && isTableEnabled())) { + while (!isTableAvailableAndInitialized()) { if (EnvironmentEdgeManager.currentTime() - startTime + 100 > timeout) { // We can't do anything if ns is not online. - throw new IOException("Timedout " + timeout + "ms waiting for namespace table to " + - "be assigned and enabled: " + getTableState()); + throw new IOException("Timedout " + timeout + "ms waiting for namespace table to " + + "be assigned and enabled: " + getTableState()); } Thread.sleep(100); } } catch (InterruptedException e) { - throw (InterruptedIOException)new InterruptedIOException().initCause(e); + throw (InterruptedIOException) new InterruptedIOException().initCause(e); } - - // initialize namespace table - isTableAvailableAndInitialized(); } private synchronized Table getNamespaceTable() throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java index 726b040..0afafe4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceAuditor.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hbase.namespace; import java.io.IOException; -import java.io.InterruptedIOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -31,7 +30,6 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.quotas.QuotaExceededException; -import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import com.google.common.annotations.VisibleForTesting; @@ -55,19 +53,6 @@ public class NamespaceAuditor { public void start() throws IOException { stateManager.start(); - long startTime = EnvironmentEdgeManager.currentTime(); - int timeout = masterServices.getConfiguration().getInt(NS_AUDITOR_INIT_TIMEOUT, - DEFAULT_NS_AUDITOR_INIT_TIMEOUT); - try { - while (!stateManager.isInitialized()) { - if (EnvironmentEdgeManager.currentTime() - startTime + 1000 > timeout) { - throw new HBaseIOException("Timed out waiting for namespace auditor to be initialized."); - } - Thread.sleep(1000); - } - } catch (InterruptedException e) { - throw (InterruptedIOException) new InterruptedIOException().initCause(e); - } LOG.info("NamespaceAuditor started."); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java index 1072157..1c8f524 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/namespace/NamespaceStateManager.java @@ -183,27 +183,22 @@ class NamespaceStateManager { /** * Initialize namespace state cache by scanning meta table. */ - void initialize() { - try { - List namespaces = this.master.listNamespaceDescriptors(); - for (NamespaceDescriptor namespace : namespaces) { - addNamespace(namespace.getName()); - List tables = this.master.listTableNamesByNamespace(namespace.getName()); - for (TableName table : tables) { - if (table.isSystemTable()) { - continue; - } - List regions = MetaTableAccessor.getTableRegions( - this.master.getConnection(), table, true); - addTable(table, regions.size()); + private void initialize() throws IOException { + List namespaces = this.master.listNamespaceDescriptors(); + for (NamespaceDescriptor namespace : namespaces) { + addNamespace(namespace.getName()); + List tables = this.master.listTableNamesByNamespace(namespace.getName()); + for (TableName table : tables) { + if (table.isSystemTable()) { + continue; } + List regions = + MetaTableAccessor.getTableRegions(this.master.getConnection(), table, true); + addTable(table, regions.size()); } - LOG.info("Finished updating state of " + nsStateCache.size() + " namespaces. "); - initialized = true; - } catch (IOException e) { - LOG.error("Error while update namespace state.", e); - initialized = false; } + LOG.info("Finished updating state of " + nsStateCache.size() + " namespaces. "); + initialized = true; } boolean isInitialized() { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java index 329430e..6204f0d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/namespace/TestNamespaceAuditor.java @@ -102,12 +102,7 @@ public class TestNamespaceAuditor { conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class, RegionServerObserver.class); UTIL.startMiniCluster(1, 1); - UTIL.waitFor(60000, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return UTIL.getHBaseCluster().getMaster().getMasterQuotaManager().isQuotaEnabled(); - } - }); + waitForQuotaEnabled(); ADMIN = UTIL.getHBaseAdmin(); } @@ -543,10 +538,7 @@ public class TestNamespaceAuditor { .getTables().size(), after.getTables().size()); } - private void restartMaster() throws Exception { - UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again"); - UTIL.getHBaseCluster().waitOnMaster(0); - UTIL.getHBaseCluster().startMaster(); + private static void waitForQuotaEnabled() throws Exception { UTIL.waitFor(60000, new Waiter.Predicate() { @Override public boolean evaluate() throws Exception { @@ -560,6 +552,13 @@ public class TestNamespaceAuditor { }); } + private void restartMaster() throws Exception { + UTIL.getHBaseCluster().getMaster(0).stop("Stopping to start again"); + UTIL.getHBaseCluster().waitOnMaster(0); + UTIL.getHBaseCluster().startMaster(); + waitForQuotaEnabled(); + } + private NamespaceAuditor getQuotaManager() { return UTIL.getHBaseCluster().getMaster() .getMasterQuotaManager().getNamespaceQuotaManager(); -- 1.9.1