diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java index 2fbfd9f..6ba517c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java @@ -628,6 +628,50 @@ public class MetaTableAccessor { return visitor.getResults(); } + /* + * @param connection + * @param excludeOfflinedSplitParents + */ + public static List> getSystemTableRegionsAndLocations( + Connection connection, final boolean excludeOfflinedSplitParents) throws IOException { + // Make a version of CollectingVisitor that collects HRegionInfo and ServerAddress + CollectingVisitor> visitor = + new CollectingVisitor>() { + private RegionLocations current = null; + + @Override + public boolean visit(Result r) throws IOException { + current = getRegionLocations(r); + if (current == null || current.getRegionLocation().getRegionInfo() == null) { + LOG.warn("No serialized HRegionInfo in " + r); + return true; + } + HRegionInfo hri = current.getRegionLocation().getRegionInfo(); + if (excludeOfflinedSplitParents && hri.isSplitParent()) return true; + + if (!hri.isSystemTable()) return true; + // Else call super and add this Result to the collection. + return super.visit(r); + } + + @Override + void add(Result r) { + if (current == null) { + return; + } + for (HRegionLocation loc : current.getRegionLocations()) { + if (loc != null) { + this.results.add(new Pair( + loc.getRegionInfo(), loc.getServerName())); + } + } + } + }; + scanMeta(connection, NamespaceDescriptor.SYSTEM_NAMESPACE_START_ROW, + NamespaceDescriptor.SYSTEM_NAMESPACE_END_ROW, QueryType.REGION, visitor); + return visitor.getResults(); + } + /** * @param connection connection we're using * @param serverName server whose regions we're interested in @@ -1074,6 +1118,7 @@ public class MetaTableAccessor { get.setTimeRange(0, time); Result result = metaHTable.get(get); + if (result == null) return null; return getTableState(result); } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/NamespaceDescriptor.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/NamespaceDescriptor.java index e1ceace..13b0568 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/NamespaceDescriptor.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/NamespaceDescriptor.java @@ -45,6 +45,10 @@ public class NamespaceDescriptor { public static final byte [] SYSTEM_NAMESPACE_NAME = Bytes.toBytes("hbase"); public static final String SYSTEM_NAMESPACE_NAME_STR = Bytes.toString(SYSTEM_NAMESPACE_NAME); + + public static final byte [] SYSTEM_NAMESPACE_START_ROW = Bytes.toBytes("hbase:"); + public static final byte [] SYSTEM_NAMESPACE_END_ROW = Bytes.toBytes("hbase;"); + /** Default namespace name. */ public static final byte [] DEFAULT_NAMESPACE_NAME = Bytes.toBytes("default"); public static final String DEFAULT_NAMESPACE_NAME_STR = diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index f7f98fe..60e9241 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -460,7 +460,7 @@ public class AssignmentManager { if (!regionsInTransition.isEmpty()) { for (RegionState regionState: regionsInTransition.values()) { ServerName serverName = regionState.getServerName(); - if (!regionState.getRegion().isMetaRegion() + if (!regionState.getRegion().isSystemTable() && serverName != null && onlineServers.contains(serverName)) { LOG.debug("Found " + regionState + " in RITs"); failover = true; @@ -509,6 +509,9 @@ public class AssignmentManager { disabledOrDisablingOrEnabling = tableStateManager.getTablesInStates( TableState.State.DISABLED, TableState.State.DISABLING, TableState.State.ENABLING); + // system tables are assigned ahead of user tables - exclude them + Set tables = ((HMaster)server).getSystemTablesBeingAssigned(); + disabledOrDisablingOrEnabling.addAll(tables); // Clean re/start, mark all user regions closed before reassignment allRegions = regionStates.closeAllUserRegions( @@ -2234,7 +2237,12 @@ public class AssignmentManager { // The region must be opening on this server. // If current state is already opened on the same server, // it could be a reportRegionTransition RPC retry. - if (current == null || !current.isOpeningOrOpenedOnServer(serverName)) { + if (current != null && current.getRegion().isSystemTable() && + !current.isOpeningOrOpenedOnServer(serverName)) { + LOG.debug("Trying to open " + current + " on " + serverName); + } + if (current == null || + (!current.getRegion().isSystemTable() && !current.isOpeningOrOpenedOnServer(serverName))) { return hri.getShortNameToLog() + " is not opening on " + serverName; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index ca721e2..65c6c8b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -245,6 +245,8 @@ public class HMaster extends HRegionServer implements MasterServices, Server { /** Namespace stuff */ private TableNamespaceManager tableNamespaceManager; + Set systemTablesAssignedBeforeInitCompletes = new HashSet(); + // Metrics for the HMaster final MetricsMaster metricsMaster; // file system manager for the master FS operations @@ -599,7 +601,10 @@ public class HMaster extends HRegionServer implements MasterServices, Server { this.mpmHost.register(new MasterFlushTableProcedureManager()); this.mpmHost.loadProcedures(conf); this.mpmHost.initialize(this, this.metricsMaster); + } + Set getSystemTablesBeingAssigned() { + return systemTablesAssignedBeforeInitCompletes; } /** @@ -730,7 +735,9 @@ public class HMaster extends HRegionServer implements MasterServices, Server { // Make sure meta assigned before proceeding. status.setStatus("Assigning Meta Region"); + systemTablesAssignedBeforeInitCompletes.clear(); assignMeta(status, previouslyFailedMetaRSs, HRegionInfo.DEFAULT_REPLICA_ID); + assignSystemTable(status); // check if master is shutting down because above assignMeta could return even hbase:meta isn't // assigned when master is shutting down if (isStopped()) return; @@ -873,6 +880,49 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } } + private void assignSystemTable(MonitoredTask status) throws IOException { + List> location = + MetaTableAccessor.getSystemTableRegionsAndLocations(getConnection(), true); + if (location == null || location.isEmpty()) return; + + String prompt; + for (int idx = 0; idx < location.size(); idx++) { + if (location.get(idx).getFirst().isMetaRegion()) { + // hbase:meta has been assigned + continue; + } + TableName table = location.get(idx).getFirst().getTable(); + if (location.get(idx).getSecond() == null) { + // there is no recorded server for the table + prompt = "Assigning " + table + " to some server"; + LOG.debug(prompt); + status.setStatus(prompt); + this.assignmentManager.assign(location.get(idx).getFirst(), true); + } else { + ServerName svr = location.get(idx).getSecond(); + if (!this.serverManager.isServerOnline(svr)) { + prompt = "Assigning " + table + " to random server since " + svr + " is not online"; + LOG.debug(prompt); + status.setStatus(prompt); + this.assignmentManager.assign(location.get(idx).getFirst(), true); + systemTablesAssignedBeforeInitCompletes.add(table); + continue; + } + List regions = new ArrayList(); + regions.add(location.get(idx).getFirst()); + prompt = "Assigning " + table + " to " + svr; + LOG.debug(prompt); + status.setStatus(prompt); + try { + this.assignmentManager.assign(svr, regions); + } catch (InterruptedException ie) { + throw new InterruptedIOException("Received " + ie.getMessage()); + } + } + systemTablesAssignedBeforeInitCompletes.add(table); + } + } + /** * Check hbase:meta is assigned. If not, assign it. * @param status MonitoredTask @@ -950,6 +1000,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { LOG.info("hbase:meta with replicaId " + replicaId + " assigned=" + assigned + ", location=" + metaTableLocator.getMetaRegionLocation(this.getZooKeeper(), replicaId)); status.setStatus("META assigned."); + systemTablesAssignedBeforeInitCompletes.add(TableName.META_TABLE_NAME); } void initNamespace() throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index e1ec225..e042e38 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -1339,8 +1339,8 @@ public class MasterRpcServices extends RSRpcServices TableName tableName = ProtobufUtil.toTableName( rt.getRegionInfo(0).getTableName()); RegionStates regionStates = master.assignmentManager.getRegionStates(); - if (!(TableName.META_TABLE_NAME.equals(tableName) - && regionStates.getRegionState(HRegionInfo.FIRST_META_REGIONINFO) != null) + if (!(tableName.isSystemTable() + && regionStates.getRegionState(HRegionInfo.convert(rt.getRegionInfo(0))) != null) && !master.assignmentManager.isFailoverCleanupDone()) { // Meta region is assigned before master finishes the // failover cleanup. So no need this check for it diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java index 02912b9..6a13a53 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/TableNamespaceManager.java @@ -134,15 +134,20 @@ public class TableNamespaceManager { } private NamespaceDescriptor get(Table table, String name) throws IOException { - Result res = table.get(new Get(Bytes.toBytes(name))); - if (res.isEmpty()) { - return null; - } - byte[] val = CellUtil.cloneValue(res.getColumnLatestCell( + try { + Result res = table.get(new Get(Bytes.toBytes(name))); + if (res.isEmpty()) { + return null; + } + byte[] val = CellUtil.cloneValue(res.getColumnLatestCell( HTableDescriptor.NAMESPACE_FAMILY_INFO_BYTES, HTableDescriptor.NAMESPACE_COL_DESC_BYTES)); - return - ProtobufUtil.toNamespaceDescriptor( + return + ProtobufUtil.toNamespaceDescriptor( HBaseProtos.NamespaceDescriptor.parseFrom(val)); + } catch (IOException ioe) { + LOG.debug("get nsd: ", ioe); + throw ioe; + } } private void create(Table table, NamespaceDescriptor ns) throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 315659a..8c63e4a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -1987,7 +1987,9 @@ public class HRegionServer extends HasThread implements if (code == TransitionCode.OPENED && openSeqNum >= 0) { transition.setOpenSeqNum(openSeqNum); } + StringBuilder sb = new StringBuilder(); for (HRegionInfo hri: hris) { + sb.append(hri + " "); transition.addRegionInfo(HRegionInfo.convert(hri)); } ReportRegionStateTransitionRequest request = builder.build(); @@ -2008,7 +2010,7 @@ public class HRegionServer extends HasThread implements return true; } catch (ServiceException se) { IOException ioe = ProtobufUtil.getRemoteException(se); - LOG.info("Failed to report region transition, will retry", ioe); + LOG.info("Failed to report region transition:" + sb.toString() + ", will retry", ioe); if (rssStub == rss) { rssStub = null; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 65cedee..15b1839 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -1514,7 +1514,9 @@ public class RSRpcServices implements HBaseRPCErrorHandler, String encodedName = region.getEncodedName(); byte[] encodedNameBytes = region.getEncodedNameAsBytes(); final Region onlineRegion = regionServer.getFromOnlineRegions(encodedName); - if (onlineRegion != null) { + if (onlineRegion != null + // && !region.isSystemTable() + ) { // The region is already online. This should not happen any more. String error = "Received OPEN for the region:" + region.getRegionNameAsString() + ", which is already online"; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java index 614f6fb..4315562 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java @@ -178,6 +178,7 @@ public class TestMasterFailover { // and he should be active active = masterThreads.get(0).getMaster(); assertNotNull(active); + active.systemTablesAssignedBeforeInitCompletes.contains(TableName.NAMESPACE_TABLE_NAME); status = active.getClusterStatus(); ServerName mastername = status.getMaster(); assertTrue(mastername.equals(active.getServerName())); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java index 692b5a0..374736c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.Threads; import org.junit.After; @@ -190,22 +191,29 @@ public class TestRestartCluster { // Wait till master is initialized and all regions are assigned RegionStates regionStates = master.getAssignmentManager().getRegionStates(); int expectedRegions = regionToRegionServerMap.size() + 1; + LOG.debug("await " + regionStates.getRegionAssignments().size() + " " + expectedRegions); + long start = EnvironmentEdgeManager.currentTime(); while (!master.isInitialized() || regionStates.getRegionAssignments().size() != expectedRegions) { Threads.sleep(100); } + LOG.debug("After wait " + (EnvironmentEdgeManager.currentTime() - start)); - snapshot =new SnapshotOfRegionAssignmentFromMeta(master.getConnection()); + snapshot = new SnapshotOfRegionAssignmentFromMeta(master.getConnection()); snapshot.initialize(); Map newRegionToRegionServerMap = snapshot.getRegionToRegionServerMap(); assertEquals(regionToRegionServerMap.size(), newRegionToRegionServerMap.size()); for (Map.Entry entry: newRegionToRegionServerMap.entrySet()) { - if (TableName.NAMESPACE_TABLE_NAME.equals(entry.getKey().getTable())) continue; + TableName table = entry.getKey().getTable(); + if (TableName.NAMESPACE_TABLE_NAME.equals(table)) continue; ServerName oldServer = regionToRegionServerMap.get(entry.getKey()); ServerName currentServer = entry.getValue(); - assertEquals(oldServer.getHostAndPort(), currentServer.getHostAndPort()); - assertNotEquals(oldServer.getStartcode(), currentServer.getStartcode()); + assertEquals(oldServer.getHostname(), currentServer.getHostname()); + assertEquals(entry.getKey() + " " + oldServer.toString(), + oldServer.getPort(), currentServer.getPort()); + assertNotEquals(entry.getKey() + " " + oldServer.toString(), oldServer.getStartcode(), + currentServer.getStartcode()); } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java index 17a208f..38a9300 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/hbck/TestOfflineMetaRebuildBase.java @@ -32,9 +32,11 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.HBaseFsck; +import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -78,6 +80,10 @@ public class TestOfflineMetaRebuildBase extends OfflineMetaRebuildTestCore { admin.enableTable(table); LOG.info("Waiting for no more RIT"); TEST_UTIL.waitUntilNoRegionsInTransition(60000); + HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); + while (master.getServerManager().areDeadServersInProgress()) { + Threads.sleep(100); + } LOG.info("No more RIT in ZK, now doing final test verification"); // everything is good again.