Index: src/main/java/org/apache/hadoop/hbase/ServerStatus.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/ServerStatus.java (revision 0) +++ src/main/java/org/apache/hadoop/hbase/ServerStatus.java (revision 0) @@ -0,0 +1,50 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hadoop.conf.Configuration; + +/** + * Set of functions that are exposed by any HBase server (implemented by the + * master and region server). + */ +public interface ServerStatus { + /** + * Return the address of the current server. + */ + public HServerAddress getHServerAddress(); + + /** + * Get the configuration object for this server. + */ + public Configuration getConfiguration(); + + // TODO: make sure the functions below work on the region server also, or get + // moved to the MasterStatus interface + public AtomicBoolean getShutdownRequested(); + + public AtomicBoolean getClosed(); + + public boolean isClosed(); + + public void shutdown(); +} Index: src/main/java/org/apache/hadoop/hbase/master/MasterStatus.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/MasterStatus.java (revision 0) +++ src/main/java/org/apache/hadoop/hbase/master/MasterStatus.java (revision 0) @@ -0,0 +1,92 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import java.io.IOException; +import java.util.List; +import java.util.Set; +import java.util.SortedMap; +import java.util.concurrent.locks.Lock; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HServerLoad; +import org.apache.hadoop.hbase.ServerStatus; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ServerConnection; +import org.apache.hadoop.hbase.ipc.HRegionInterface; +import org.apache.hadoop.hbase.master.metrics.MasterMetrics; +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper; + +/** + * These are the set of functions implemented by the HMaster and accessed by + * the other packages in the master. + * + * TODO: this list has to be cleaned up, this is a re-factor only change that + * preserves the functions in the interface. + */ +public interface MasterStatus extends ServerStatus { + + public ServerManager getServerManager(); + + public RegionManager getRegionManager(); + + public boolean isClusterStartup(); + + public FileSystem getFileSystem(); + + public Path getOldLogDir(); + + public ZooKeeperWrapper getZooKeeperWrapper(); + + public void startShutdown(); + + public MasterMetrics getMetrics(); + + public RegionServerOperationQueue getRegionServerOperationQueue(); + + public ServerConnection getServerConnection(); + + public boolean checkFileSystem(); + + public int getThreadWakeFrequency(); + + public int getNumRetries(); + + public void deleteEmptyMetaRows(HRegionInterface s, + byte [] metaRegionName, + List emptyRows); + + public HRegionInfo getHRegionInfo(final byte [] row, final Result res) throws IOException; + + public Path getRootDir(); + + public Lock getSplitLogLock(); + + public int numServers(); + + public void getLightServers(final HServerLoad l, + SortedMap> m); + + public SortedMap> getLoadToServers(); + + public double getAverageLoad(); +} Index: src/main/java/org/apache/hadoop/hbase/master/MetaScanner.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/MetaScanner.java (revision 957672) +++ src/main/java/org/apache/hadoop/hbase/master/MetaScanner.java (working copy) @@ -50,20 +50,20 @@ * * @param master */ - public MetaScanner(HMaster master) { - super(master, false, master.getShutdownRequested()); + public MetaScanner(MasterStatus masterStatus) { + super(masterStatus, false, masterStatus.getShutdownRequested()); } // Don't retry if we get an error while scanning. Errors are most often // caused by the server going away. Wait until next rescan interval when // things should be back to normal. private boolean scanOneMetaRegion(MetaRegion region) { - while (!this.master.isClosed() && - !this.master.getRegionManager().isInitialRootScanComplete() && - this.master.getRegionManager().getRootRegionLocation() == null) { + while (!this.masterStatus.isClosed() && + !this.masterStatus.getRegionManager().isInitialRootScanComplete() && + this.masterStatus.getRegionManager().getRootRegionLocation() == null) { sleep(); } - if (this.master.isClosed()) { + if (this.masterStatus.isClosed()) { return false; } @@ -71,7 +71,7 @@ // Don't interrupt us while we're working synchronized (scannerLock) { scanRegion(region); - this.master.getRegionManager().putMetaRegionOnline(region); + this.masterStatus.getRegionManager().putMetaRegionOnline(region); } } catch (IOException e) { e = RemoteExceptionHandler.checkIOException(e); @@ -80,13 +80,13 @@ // so, either it won't be in the onlineMetaRegions list or its host // address has changed and the containsValue will fail. If not // found, best thing to do here is probably return. - if (!this.master.getRegionManager().isMetaRegionOnline(region.getStartKey())) { + if (!this.masterStatus.getRegionManager().isMetaRegionOnline(region.getStartKey())) { LOG.debug("Scanned region is no longer in map of online " + "regions or its value has changed"); return false; } // Make sure the file system is still available - this.master.checkFileSystem(); + this.masterStatus.checkFileSystem(); } catch (Exception e) { // If for some reason we get some other kind of exception, // at least log it rather than go out silently. @@ -98,11 +98,11 @@ @Override protected boolean initialScan() { MetaRegion region = null; - while (!this.master.isClosed() && + while (!this.masterStatus.isClosed() && (region == null && metaRegionsToScan.size() > 0) && !metaRegionsScanned()) { try { - region = metaRegionsToScan.poll(this.master.getThreadWakeFrequency(), + region = metaRegionsToScan.poll(this.masterStatus.getThreadWakeFrequency(), TimeUnit.MILLISECONDS); } catch (InterruptedException e) { // continue @@ -123,7 +123,7 @@ @Override protected void maintenanceScan() { List regions = - this.master.getRegionManager().getListOfOnlineMetaRegions(); + this.masterStatus.getRegionManager().getListOfOnlineMetaRegions(); int regionCount = 0; for (MetaRegion r: regions) { scanOneMetaRegion(r); @@ -141,9 +141,9 @@ * @return False if number of meta regions matches count of online regions. */ private synchronized boolean metaRegionsScanned() { - if (!this.master.getRegionManager().isInitialRootScanComplete() || - this.master.getRegionManager().numMetaRegions() != - this.master.getRegionManager().numOnlineMetaRegions()) { + if (!this.masterStatus.getRegionManager().isInitialRootScanComplete() || + this.masterStatus.getRegionManager().numMetaRegions() != + this.masterStatus.getRegionManager().numOnlineMetaRegions()) { return false; } notifyAll(); @@ -155,21 +155,21 @@ * been scanned. */ synchronized boolean waitForMetaRegionsOrClose() { - while (!this.master.isClosed()) { - synchronized (master.getRegionManager()) { - if (this.master.getRegionManager().isInitialRootScanComplete() && - this.master.getRegionManager().numMetaRegions() == - this.master.getRegionManager().numOnlineMetaRegions()) { + while (!this.masterStatus.isClosed()) { + synchronized (masterStatus.getRegionManager()) { + if (this.masterStatus.getRegionManager().isInitialRootScanComplete() && + this.masterStatus.getRegionManager().numMetaRegions() == + this.masterStatus.getRegionManager().numOnlineMetaRegions()) { break; } } try { - wait(this.master.getThreadWakeFrequency()); + wait(this.masterStatus.getThreadWakeFrequency()); } catch (InterruptedException e) { // continue } } - return this.master.isClosed(); + return this.masterStatus.isClosed(); } /** Index: src/main/java/org/apache/hadoop/hbase/master/ServerManager.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (revision 957672) +++ src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (working copy) @@ -85,7 +85,7 @@ private final Map serversToLoad = new ConcurrentHashMap(); - private HMaster master; + private MasterStatus masterStatus; /* The regionserver will not be assigned or asked close regions if it * is currently opening >= this many regions. @@ -136,24 +136,24 @@ /** * Constructor. - * @param master + * @param masterStatus */ - public ServerManager(HMaster master) { - this.master = master; - Configuration c = master.getConfiguration(); + public ServerManager(MasterStatus masterStatus) { + this.masterStatus = masterStatus; + Configuration c = masterStatus.getConfiguration(); this.nobalancingCount = c.getInt("hbase.regions.nobalancing.count", 4); int metaRescanInterval = c.getInt("hbase.master.meta.thread.rescanfrequency", 60 * 1000); this.minimumServerCount = c.getInt("hbase.regions.server.count.min", 0); this.serverMonitorThread = new ServerMonitor(metaRescanInterval, - this.master.getShutdownRequested()); + this.masterStatus.getShutdownRequested()); String n = Thread.currentThread().getName(); Threads.setDaemonThreadRunning(this.serverMonitorThread, n + ".serverMonitor"); this.oldLogCleaner = new OldLogsCleaner( c.getInt("hbase.master.meta.thread.rescanfrequency",60 * 1000), - this.master.getShutdownRequested(), c, - master.getFileSystem(), master.getOldLogDir()); + this.masterStatus.getShutdownRequested(), c, + masterStatus.getFileSystem(), masterStatus.getOldLogDir()); Threads.setDaemonThreadRunning(oldLogCleaner, n + ".oldLogCleaner"); @@ -238,7 +238,7 @@ // We must set this watcher here because it can be set on a fresh start // or on a failover Watcher watcher = new ServerExpirer(new HServerInfo(info)); - this.master.getZooKeeperWrapper().updateRSLocationGetWatch(info, watcher); + this.masterStatus.getZooKeeperWrapper().updateRSLocationGetWatch(info, watcher); this.serversToServerInfo.put(serverName, info); this.serversToLoad.put(serverName, load); synchronized (this.loadToServers) { @@ -278,14 +278,14 @@ this.quiescedServers.incrementAndGet(); } } - if (this.master.getShutdownRequested().get()) { + if (this.masterStatus.getShutdownRequested().get()) { if (quiescedServers.get() >= serversToServerInfo.size()) { // If the only servers we know about are meta servers, then we can // proceed with shutdown LOG.info("All user tables quiesced. Proceeding with shutdown"); - this.master.startShutdown(); + this.masterStatus.startShutdown(); } - if (!this.master.isClosed()) { + if (!this.masterStatus.isClosed()) { if (msgs.length > 0 && msgs[0].isType(HMsg.Type.MSG_REPORT_QUIESCED)) { // Server is already quiesced, but we aren't ready to shut down @@ -296,7 +296,7 @@ return new HMsg [] {HMsg.REGIONSERVER_QUIESCE}; } } - if (this.master.isClosed()) { + if (this.masterStatus.isClosed()) { // Tell server to shut down if we are shutting down. This should // happen after check of MSG_REPORT_EXITING above, since region server // will send us one of these messages after it gets MSG_REGIONSERVER_STOP @@ -355,7 +355,7 @@ ": MSG_REPORT_EXITING"); // Get all the regions the server was serving reassigned // (if we are not shutting down). - if (!master.closed.get()) { + if (!masterStatus.getClosed().get()) { for (int i = 1; i < msgs.length; i++) { LOG.info("Processing " + msgs[i] + " from " + serverInfo.getServerName()); @@ -363,11 +363,11 @@ HRegionInfo info = msgs[i].getRegionInfo(); // Meta/root region offlining is handed in removeServerInfo above. if (!info.isMetaRegion()) { - synchronized (master.getRegionManager()) { - if (!master.getRegionManager().isOfflined(info.getRegionNameAsString())) { - master.getRegionManager().setUnassigned(info, true); + synchronized (masterStatus.getRegionManager()) { + if (!masterStatus.getRegionManager().isOfflined(info.getRegionNameAsString())) { + masterStatus.getRegionManager().setUnassigned(info, true); } else { - master.getRegionManager().removeRegion(info); + masterStatus.getRegionManager().removeRegion(info); } } } @@ -375,13 +375,13 @@ } // There should not be any regions in transition for this server - the // server should finish transitions itself before closing - Map inTransition = master.getRegionManager() + Map inTransition = masterStatus.getRegionManager() .getRegionsInTransitionOnServer(serverInfo.getServerName()); for (Map.Entry entry : inTransition.entrySet()) { LOG.warn("Region server " + serverInfo.getServerName() + " shut down with region " + entry.getKey() + " in transition " + "state " + entry.getValue()); - master.getRegionManager().setUnassigned(entry.getValue().getRegionInfo(), + masterStatus.getRegionManager().setUnassigned(entry.getValue().getRegionInfo(), true); } } @@ -403,7 +403,7 @@ this.serversToServerInfo.put(serverInfo.getServerName(), serverInfo); HServerLoad load = this.serversToLoad.get(serverInfo.getServerName()); if (load != null) { - this.master.getMetrics().incrementRequests(load.getNumberOfRequests()); + this.masterStatus.getMetrics().incrementRequests(load.getNumberOfRequests()); if (!load.equals(serverInfo.getLoad())) { updateLoadToServers(serverInfo.getServerName(), load); } @@ -450,7 +450,7 @@ LOG.info("Processing " + incomingMsgs[i] + " from " + serverInfo.getServerName() + "; " + (i + 1) + " of " + incomingMsgs.length); - if (!this.master.getRegionServerOperationQueue(). + if (!this.masterStatus.getRegionServerOperationQueue(). process(serverInfo, incomingMsgs[i])) { continue; } @@ -483,13 +483,13 @@ } } - synchronized (this.master.getRegionManager()) { + synchronized (this.masterStatus.getRegionManager()) { // Tell the region server to close regions that we have marked for closing. for (HRegionInfo i: - this.master.getRegionManager().getMarkedToClose(serverInfo.getServerName())) { + this.masterStatus.getRegionManager().getMarkedToClose(serverInfo.getServerName())) { returnMsgs.add(new HMsg(HMsg.Type.MSG_REGION_CLOSE, i)); // Transition the region from toClose to closing state - this.master.getRegionManager().setPendingClose(i.getRegionNameAsString()); + this.masterStatus.getRegionManager().setPendingClose(i.getRegionNameAsString()); } // Figure out what the RegionServer ought to do, and write back. @@ -497,12 +497,12 @@ // Should we tell it close regions because its overloaded? If its // currently opening regions, leave it alone till all are open. if (openingCount < this.nobalancingCount) { - this.master.getRegionManager().assignRegions(serverInfo, mostLoadedRegions, + this.masterStatus.getRegionManager().assignRegions(serverInfo, mostLoadedRegions, returnMsgs); } // Send any pending table actions. - this.master.getRegionManager().applyActions(serverInfo, returnMsgs); + this.masterStatus.getRegionManager().applyActions(serverInfo, returnMsgs); } return returnMsgs.toArray(new HMsg[returnMsgs.size()]); } @@ -516,17 +516,17 @@ * @param returnMsgs */ private void processSplitRegion(HRegionInfo region, HRegionInfo a, HRegionInfo b) { - synchronized (master.getRegionManager()) { + synchronized (masterStatus.getRegionManager()) { // Cancel any actions pending for the affected region. // This prevents the master from sending a SPLIT message if the table // has already split by the region server. - this.master.getRegionManager().endActions(region.getRegionName()); + this.masterStatus.getRegionManager().endActions(region.getRegionName()); assignSplitDaughter(a); assignSplitDaughter(b); if (region.isMetaTable()) { // A meta region has split. - this. master.getRegionManager().offlineMetaRegionWithStartKey(region.getStartKey()); - this.master.getRegionManager().incrementNumMetaRegions(); + this. masterStatus.getRegionManager().offlineMetaRegionWithStartKey(region.getStartKey()); + this.masterStatus.getRegionManager().incrementNumMetaRegions(); } } } @@ -541,12 +541,12 @@ */ private void assignSplitDaughter(final HRegionInfo hri) { MetaRegion mr = - this.master.getRegionManager().getFirstMetaRegionForRegion(hri); + this.masterStatus.getRegionManager().getFirstMetaRegionForRegion(hri); Get g = new Get(hri.getRegionName()); g.addFamily(HConstants.CATALOG_FAMILY); try { HRegionInterface server = - this.master.getServerConnection().getHRegionConnection(mr.getServer()); + this.masterStatus.getServerConnection().getHRegionConnection(mr.getServer()); Result r = server.get(mr.getRegionName(), g); // If size > 3 -- presume regioninfo, startcode and server -- then presume // that this daughter already assigned and return. @@ -555,7 +555,7 @@ LOG.warn("Failed get on " + HConstants.CATALOG_FAMILY_STR + "; possible double-assignment?", e); } - this.master.getRegionManager().setUnassigned(hri, false); + this.masterStatus.getRegionManager().setUnassigned(hri, false); } /* @@ -567,13 +567,13 @@ public void processRegionOpen(HServerInfo serverInfo, HRegionInfo region, ArrayList returnMsgs) { boolean duplicateAssignment = false; - synchronized (master.getRegionManager()) { - if (!this.master.getRegionManager().isUnassigned(region) && - !this.master.getRegionManager().isPendingOpen(region.getRegionNameAsString())) { + synchronized (masterStatus.getRegionManager()) { + if (!this.masterStatus.getRegionManager().isUnassigned(region) && + !this.masterStatus.getRegionManager().isPendingOpen(region.getRegionNameAsString())) { if (region.isRootRegion()) { // Root region HServerAddress rootServer = - this.master.getRegionManager().getRootRegionLocation(); + this.masterStatus.getRegionManager().getRootRegionLocation(); if (rootServer != null) { if (rootServer.compareTo(serverInfo.getServerAddress()) == 0) { // A duplicate open report from the correct server @@ -587,7 +587,7 @@ // Not root region. If it is not a pending region, then we are // going to treat it as a duplicate assignment, although we can't // tell for certain that's the case. - if (this.master.getRegionManager().isPendingOpen( + if (this.masterStatus.getRegionManager().isPendingOpen( region.getRegionNameAsString())) { // A duplicate report from the correct server return; @@ -610,20 +610,20 @@ if (region.isRootRegion()) { // it was assigned, and it's not a duplicate assignment, so take it out // of the unassigned list. - this.master.getRegionManager().removeRegion(region); + this.masterStatus.getRegionManager().removeRegion(region); // Store the Root Region location (in memory) HServerAddress rootServer = serverInfo.getServerAddress(); - this.master.getServerConnection().setRootRegionLocation( + this.masterStatus.getServerConnection().setRootRegionLocation( new HRegionLocation(region, rootServer)); - this.master.getRegionManager().setRootRegionLocation(rootServer); + this.masterStatus.getRegionManager().setRootRegionLocation(rootServer); } else { // Note that the table has been assigned and is waiting for the // meta table to be updated. - this.master.getRegionManager().setOpen(region.getRegionNameAsString()); + this.masterStatus.getRegionManager().setOpen(region.getRegionNameAsString()); RegionServerOperation op = - new ProcessRegionOpen(master, serverInfo, region); - this.master.getRegionServerOperationQueue().put(op); + new ProcessRegionOpen(masterStatus, serverInfo, region); + this.masterStatus.getRegionServerOperationQueue().put(op); } } } @@ -634,24 +634,24 @@ * @throws Exception */ public void processRegionClose(HRegionInfo region) { - synchronized (this.master.getRegionManager()) { + synchronized (this.masterStatus.getRegionManager()) { if (region.isRootRegion()) { // Root region - this.master.getRegionManager().unsetRootRegion(); + this.masterStatus.getRegionManager().unsetRootRegion(); if (region.isOffline()) { // Can't proceed without root region. Shutdown. LOG.fatal("root region is marked offline"); - this.master.shutdown(); + this.masterStatus.shutdown(); return; } } else if (region.isMetaTable()) { // Region is part of the meta table. Remove it from onlineMetaRegions - this.master.getRegionManager().offlineMetaRegionWithStartKey(region.getStartKey()); + this.masterStatus.getRegionManager().offlineMetaRegionWithStartKey(region.getStartKey()); } boolean offlineRegion = - this.master.getRegionManager().isOfflined(region.getRegionNameAsString()); + this.masterStatus.getRegionManager().isOfflined(region.getRegionNameAsString()); boolean reassignRegion = !region.isOffline() && !offlineRegion; // NOTE: If the region was just being closed and not offlined, we cannot @@ -659,10 +659,10 @@ // the messages we've received. In this case, a close could be // processed before an open resulting in the master not agreeing on // the region's state. - this.master.getRegionManager().setClosed(region.getRegionNameAsString()); + this.masterStatus.getRegionManager().setClosed(region.getRegionNameAsString()); RegionServerOperation op = - new ProcessRegionClose(master, region, offlineRegion, reassignRegion); - this.master.getRegionServerOperationQueue().put(op); + new ProcessRegionClose(masterStatus, region, offlineRegion, reassignRegion); + this.masterStatus.getRegionServerOperationQueue().put(op); } } @@ -674,18 +674,18 @@ // This method can be called a couple of times during shutdown. if (info != null) { LOG.info("Removing server's info " + serverName); - this.master.getRegionManager().offlineMetaServer(info.getServerAddress()); + this.masterStatus.getRegionManager().offlineMetaServer(info.getServerAddress()); //HBASE-1928: Check whether this server has been transitioning the ROOT table - if (this.master.getRegionManager().isRootInTransitionOnThisServer(serverName)) { - this.master.getRegionManager().unsetRootRegion(); - this.master.getRegionManager().reassignRootRegion(); + if (this.masterStatus.getRegionManager().isRootInTransitionOnThisServer(serverName)) { + this.masterStatus.getRegionManager().unsetRootRegion(); + this.masterStatus.getRegionManager().reassignRootRegion(); } //HBASE-1928: Check whether this server has been transitioning the META table - HRegionInfo metaServerRegionInfo = this.master.getRegionManager().getMetaServerRegionInfo (serverName); + HRegionInfo metaServerRegionInfo = this.masterStatus.getRegionManager().getMetaServerRegionInfo (serverName); if (metaServerRegionInfo != null) { - this.master.getRegionManager().setUnassigned(metaServerRegionInfo, true); + this.masterStatus.getRegionManager().setUnassigned(metaServerRegionInfo, true); } infoUpdated = true; @@ -801,7 +801,7 @@ * a MSG_REGIONSERVER_STOP. */ void letRegionServersShutdown() { - if (!master.checkFileSystem()) { + if (!masterStatus.checkFileSystem()) { // Forget waiting for the region servers if the file system has gone // away. Just exit as quickly as possible. return; @@ -870,8 +870,8 @@ LOG.debug("Added=" + serverName + " to dead servers, added shutdown processing operation"); this.deadServers.add(serverName); - this.master.getRegionServerOperationQueue(). - put(new ProcessServerShutdown(master, info)); + this.masterStatus.getRegionServerOperationQueue(). + put(new ProcessServerShutdown(masterStatus, info)); } /** Index: src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java (revision 957672) +++ src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java (working copy) @@ -39,13 +39,13 @@ protected final HServerInfo serverInfo; /** - * @param master + * @param masterStatus * @param info * @param regionInfo */ - public ProcessRegionOpen(HMaster master, HServerInfo info, + public ProcessRegionOpen(MasterStatus masterStatus, HServerInfo info, HRegionInfo regionInfo) { - super(master, regionInfo); + super(masterStatus, regionInfo); if (info == null) { throw new NullPointerException("HServerInfo cannot be null; " + "hbase-958 debugging"); @@ -69,7 +69,7 @@ return true; } HRegionInterface server = - master.getServerConnection().getHRegionConnection(getMetaRegion().getServer()); + masterStatus.getServerConnection().getHRegionConnection(getMetaRegion().getServer()); LOG.info(regionInfo.getRegionNameAsString() + " open on " + serverInfo.getServerName()); @@ -83,40 +83,40 @@ LOG.info("Updated row " + regionInfo.getRegionNameAsString() + " in region " + Bytes.toString(metaRegionName) + " with startcode=" + serverInfo.getStartCode() + ", server=" + serverInfo.getHostnamePort()); - synchronized (master.getRegionManager()) { + synchronized (masterStatus.getRegionManager()) { if (isMetaTable) { // It's a meta region. MetaRegion m = new MetaRegion(new HServerAddress(serverInfo.getServerAddress()), regionInfo); - if (!master.getRegionManager().isInitialMetaScanComplete()) { + if (!masterStatus.getRegionManager().isInitialMetaScanComplete()) { // Put it on the queue to be scanned for the first time. if (LOG.isDebugEnabled()) { LOG.debug("Adding " + m.toString() + " to regions to scan"); } - master.getRegionManager().addMetaRegionToScan(m); + masterStatus.getRegionManager().addMetaRegionToScan(m); } else { // Add it to the online meta regions if (LOG.isDebugEnabled()) { LOG.debug("Adding to onlineMetaRegions: " + m.toString()); } - master.getRegionManager().putMetaRegionOnline(m); + masterStatus.getRegionManager().putMetaRegionOnline(m); // Interrupting the Meta Scanner sleep so that it can // process regions right away - master.getRegionManager().metaScannerThread.triggerNow(); + masterStatus.getRegionManager().metaScannerThread.triggerNow(); } } // If updated successfully, remove from pending list if the state // is consistent. For example, a disable could be called before the // synchronization. - if(master.getRegionManager(). + if(masterStatus.getRegionManager(). isOfflined(regionInfo.getRegionNameAsString())) { LOG.warn("We opened a region while it was asked to be closed."); } else { - master.getRegionManager().removeRegion(regionInfo); + masterStatus.getRegionManager().removeRegion(regionInfo); } ZooKeeperWrapper zkWrapper = - ZooKeeperWrapper.getInstance(master.getConfiguration(), + ZooKeeperWrapper.getInstance(masterStatus.getConfiguration(), HMaster.class.getName()); zkWrapper.deleteUnassignedRegion(regionInfo.getEncodedName()); return true; Index: src/main/java/org/apache/hadoop/hbase/master/TableOperation.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/TableOperation.java (revision 957672) +++ src/main/java/org/apache/hadoop/hbase/master/TableOperation.java (working copy) @@ -84,7 +84,7 @@ final Scan scan = new Scan(tableNameMetaStart) .addFamily(HConstants.CATALOG_FAMILY); long scannerId = this.server.openScanner(m.getRegionName(), scan); - int rows = this.master.getConfiguration(). + int rows = this.masterStatus.getConfiguration(). getInt("hbase.meta.scanner.caching", 100); scan.setCaching(rows); List emptyRows = new ArrayList(); @@ -94,7 +94,7 @@ if (values == null || values.isEmpty()) { break; } - HRegionInfo info = this.master.getHRegionInfo(values.getRow(), values); + HRegionInfo info = this.masterStatus.getHRegionInfo(values.getRow(), values); if (info == null) { emptyRows.add(values.getRow()); LOG.error(Bytes.toString(HConstants.CATALOG_FAMILY) + ":" @@ -137,7 +137,7 @@ LOG.warn("Found " + emptyRows.size() + " rows with empty HRegionInfo while scanning meta region " + Bytes.toString(m.getRegionName())); - master.deleteEmptyMetaRows(server, m.getRegionName(), emptyRows); + masterStatus.deleteEmptyMetaRows(server, m.getRegionName(), emptyRows); } if (!tableExists) { Index: src/main/java/org/apache/hadoop/hbase/master/RetryableMetaOperation.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/RetryableMetaOperation.java (revision 957672) +++ src/main/java/org/apache/hadoop/hbase/master/RetryableMetaOperation.java (working copy) @@ -43,27 +43,27 @@ protected final Log LOG = LogFactory.getLog(this.getClass()); protected final Sleeper sleeper; protected final MetaRegion m; - protected final HMaster master; + protected final MasterStatus masterStatus; protected HRegionInterface server; - protected RetryableMetaOperation(MetaRegion m, HMaster master) { + protected RetryableMetaOperation(MetaRegion m, MasterStatus masterStatus) { this.m = m; - this.master = master; - this.sleeper = new Sleeper(this.master.getThreadWakeFrequency(), - this.master.getClosed()); + this.masterStatus = masterStatus; + this.sleeper = new Sleeper(this.masterStatus.getThreadWakeFrequency(), + this.masterStatus.getClosed()); } protected T doWithRetries() throws IOException, RuntimeException { List exceptions = new ArrayList(); - for (int tries = 0; tries < this.master.getNumRetries(); tries++) { - if (this.master.isClosed()) { + for (int tries = 0; tries < this.masterStatus.getNumRetries(); tries++) { + if (this.masterStatus.isClosed()) { return null; } try { this.server = - this.master.getServerConnection().getHRegionConnection(m.getServer()); + this.masterStatus.getServerConnection().getHRegionConnection(m.getServer()); return this.call(); } catch (IOException e) { if (e instanceof TableNotFoundException || @@ -74,7 +74,7 @@ if (e instanceof RemoteException) { e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e); } - if (tries == this.master.getNumRetries() - 1) { + if (tries == this.masterStatus.getNumRetries() - 1) { if (LOG.isDebugEnabled()) { StringBuilder message = new StringBuilder( "Trying to contact region server for regionName '" + @@ -86,7 +86,7 @@ } LOG.debug(message); } - this.master.checkFileSystem(); + this.masterStatus.checkFileSystem(); throw e; } if (LOG.isDebugEnabled()) { Index: src/main/java/org/apache/hadoop/hbase/master/ZKUnassignedWatcher.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/ZKUnassignedWatcher.java (revision 957672) +++ src/main/java/org/apache/hadoop/hbase/master/ZKUnassignedWatcher.java (working copy) @@ -46,22 +46,22 @@ String serverName; ServerManager serverManager; - public static void start(Configuration conf, HMaster master) + public static void start(Configuration conf, MasterStatus masterStatus) throws IOException { - new ZKUnassignedWatcher(conf, master); + new ZKUnassignedWatcher(conf, masterStatus); LOG.debug("Started ZKUnassigned watcher"); } - public ZKUnassignedWatcher(Configuration conf, HMaster master) + public ZKUnassignedWatcher(Configuration conf, MasterStatus masterStatus) throws IOException { - this.serverName = master.getHServerAddress().toString(); - this.serverManager = master.getServerManager(); + this.serverName = masterStatus.getHServerAddress().toString(); + this.serverManager = masterStatus.getServerManager(); zkWrapper = ZooKeeperWrapper.getInstance(conf, HMaster.class.getName()); String unassignedZNode = zkWrapper.getRegionInTransitionZNode(); // If the UNASSIGNED ZNode exists and this is a fresh cluster start, then // delete it. - if(master.isClusterStartup() && zkWrapper.exists(unassignedZNode, false)) { + if(masterStatus.isClusterStartup() && zkWrapper.exists(unassignedZNode, false)) { LOG.info("Cluster start, but found " + unassignedZNode + ", deleting it."); try { zkWrapper.deleteZNode(unassignedZNode, true); Index: src/main/java/org/apache/hadoop/hbase/master/RegionServerOperation.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/RegionServerOperation.java (revision 957672) +++ src/main/java/org/apache/hadoop/hbase/master/RegionServerOperation.java (working copy) @@ -31,14 +31,14 @@ LogFactory.getLog(RegionServerOperation.class.getName()); private long expire; - protected final HMaster master; + protected final MasterStatus masterStatus; /* How long we stay on queue. */ private int delay; - protected RegionServerOperation(HMaster master) { - this.master = master; - this.delay = this.master.getConfiguration(). + protected RegionServerOperation(MasterStatus masterStatus) { + this.masterStatus = masterStatus; + this.delay = this.masterStatus.getConfiguration(). getInt("hbase.server.thread.wakefrequency", 10 * 1000); // Set the future time at which we expect to be released from the // DelayQueue we're inserted in on lease expiration. @@ -71,7 +71,7 @@ } protected void requeue() { - this.master.getRegionServerOperationQueue().putOnDelayQueue(this); + this.masterStatus.getRegionServerOperationQueue().putOnDelayQueue(this); } private long whenToExpire() { @@ -80,7 +80,7 @@ protected boolean rootAvailable() { boolean available = true; - if (this.master.getRegionManager().getRootRegionLocation() == null) { + if (this.masterStatus.getRegionManager().getRootRegionLocation() == null) { available = false; requeue(); } @@ -89,9 +89,9 @@ protected boolean metaTableAvailable() { boolean available = true; - if ((master.getRegionManager().numMetaRegions() != - master.getRegionManager().numOnlineMetaRegions()) || - master.getRegionManager().metaRegionsInTransition()) { + if ((masterStatus.getRegionManager().numMetaRegions() != + masterStatus.getRegionManager().numOnlineMetaRegions()) || + masterStatus.getRegionManager().metaRegionsInTransition()) { // We can't proceed because not all of the meta regions are online. // We can't block either because that would prevent the meta region // online message from being processed. In order to prevent spinning @@ -99,9 +99,9 @@ // other threads the opportunity to get the meta regions on-line. if (LOG.isDebugEnabled()) { LOG.debug("numberOfMetaRegions: " + - master.getRegionManager().numMetaRegions() + + masterStatus.getRegionManager().numMetaRegions() + ", onlineMetaRegions.size(): " + - master.getRegionManager().numOnlineMetaRegions()); + masterStatus.getRegionManager().numOnlineMetaRegions()); LOG.debug("Requeuing because not all meta regions are online"); } available = false; Index: src/main/java/org/apache/hadoop/hbase/master/RootScanner.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/RootScanner.java (revision 957672) +++ src/main/java/org/apache/hadoop/hbase/master/RootScanner.java (working copy) @@ -28,10 +28,10 @@ class RootScanner extends BaseScanner { /** * Constructor - * @param master + * @param masterStatus */ - public RootScanner(HMaster master) { - super(master, true, master.getShutdownRequested()); + public RootScanner(MasterStatus masterStatus) { + super(masterStatus, true, masterStatus.getShutdownRequested()); } /** @@ -42,16 +42,16 @@ * @return True if successfully scanned. */ private boolean scanRoot() { - master.getRegionManager().waitForRootRegionLocation(); - if (master.isClosed()) { + masterStatus.getRegionManager().waitForRootRegionLocation(); + if (masterStatus.isClosed()) { return false; } try { // Don't interrupt us while we're working synchronized(scannerLock) { - if (master.getRegionManager().getRootRegionLocation() != null) { - scanRegion(new MetaRegion(master.getRegionManager().getRootRegionLocation(), + if (masterStatus.getRegionManager().getRootRegionLocation() != null) { + scanRegion(new MetaRegion(masterStatus.getRegionManager().getRootRegionLocation(), HRegionInfo.ROOT_REGIONINFO)); } } @@ -59,7 +59,7 @@ e = RemoteExceptionHandler.checkIOException(e); LOG.warn("Scan ROOT region", e); // Make sure the file system is still available - master.checkFileSystem(); + masterStatus.checkFileSystem(); } catch (Exception e) { // If for some reason we get some other kind of exception, // at least log it rather than go out silently. Index: src/main/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java (revision 957672) +++ src/main/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java (working copy) @@ -68,17 +68,17 @@ } /** - * @param master + * @param masterStatus * @param serverInfo */ - public ProcessServerShutdown(HMaster master, HServerInfo serverInfo) { - super(master); + public ProcessServerShutdown(MasterStatus masterStatus, HServerInfo serverInfo) { + super(masterStatus); this.deadServer = serverInfo.getServerName(); this.deadServerAddress = serverInfo.getServerAddress(); this.logSplit = false; this.rootRescanned = false; this.rsLogDir = - new Path(master.getRootDir(), HLog.getHLogDirectoryName(serverInfo)); + new Path(masterStatus.getRootDir(), HLog.getHLogDirectoryName(serverInfo)); // check to see if I am responsible for either ROOT or any of the META tables. @@ -88,22 +88,22 @@ private void closeMetaRegions() { this.isRootServer = - this.master.getRegionManager().isRootServer(this.deadServerAddress) || - this.master.getRegionManager().isRootInTransitionOnThisServer(deadServer); + this.masterStatus.getRegionManager().isRootServer(this.deadServerAddress) || + this.masterStatus.getRegionManager().isRootInTransitionOnThisServer(deadServer); if (this.isRootServer) { - this.master.getRegionManager().unsetRootRegion(); + this.masterStatus.getRegionManager().unsetRootRegion(); } List metaStarts = - this.master.getRegionManager().listMetaRegionsForServer(deadServerAddress); + this.masterStatus.getRegionManager().listMetaRegionsForServer(deadServerAddress); this.metaRegions = new ArrayList(); for (byte [] startKey: metaStarts) { - MetaRegion r = master.getRegionManager().offlineMetaRegionWithStartKey(startKey); + MetaRegion r = masterStatus.getRegionManager().offlineMetaRegionWithStartKey(startKey); this.metaRegions.add(r); } //HBASE-1928: Check whether this server has been transitioning the META table - HRegionInfo metaServerRegionInfo = master.getRegionManager().getMetaServerRegionInfo (deadServer); + HRegionInfo metaServerRegionInfo = masterStatus.getRegionManager().getMetaServerRegionInfo (deadServer); if (metaServerRegionInfo != null) { metaRegions.add (new MetaRegion (deadServerAddress, metaServerRegionInfo)); } @@ -118,14 +118,14 @@ private void closeRegionsInTransition() { Map inTransition = - master.getRegionManager().getRegionsInTransitionOnServer(deadServer); + masterStatus.getRegionManager().getRegionsInTransitionOnServer(deadServer); for (Map.Entry entry : inTransition.entrySet()) { String regionName = entry.getKey(); RegionState state = entry.getValue(); LOG.info("Region " + regionName + " was in transition " + state + " on dead server " + deadServer + " - marking unassigned"); - master.getRegionManager().setUnassigned(state.getRegionInfo(), true); + masterStatus.getRegionManager().setUnassigned(state.getRegionInfo(), true); } } @@ -177,28 +177,28 @@ Bytes.toString(row)); } - HRegionInfo info = master.getHRegionInfo(row, values); + HRegionInfo info = masterStatus.getHRegionInfo(row, values); if (info == null) { emptyRows.add(row); continue; } - synchronized (master.getRegionManager()) { + synchronized (masterStatus.getRegionManager()) { if (info.isMetaTable()) { if (LOG.isDebugEnabled()) { LOG.debug("removing meta region " + Bytes.toString(info.getRegionName()) + " from online meta regions"); } - master.getRegionManager().offlineMetaRegionWithStartKey(info.getStartKey()); + masterStatus.getRegionManager().offlineMetaRegionWithStartKey(info.getStartKey()); } ToDoEntry todo = new ToDoEntry(info); toDoList.add(todo); - if (master.getRegionManager().isOfflined(info.getRegionNameAsString()) || + if (masterStatus.getRegionManager().isOfflined(info.getRegionNameAsString()) || info.isOffline()) { - master.getRegionManager().removeRegion(info); + masterStatus.getRegionManager().removeRegion(info); // Mark region offline if (!info.isOffline()) { todo.regionOffline = true; @@ -228,7 +228,7 @@ LOG.warn("Found " + emptyRows.size() + " rows with empty HRegionInfo while scanning meta region " + Bytes.toString(regionName)); - master.deleteEmptyMetaRows(server, regionName, emptyRows); + masterStatus.deleteEmptyMetaRows(server, regionName, emptyRows); } // Update server in root/meta entries for (ToDoEntry e: toDoList) { @@ -239,19 +239,19 @@ // Get regions reassigned for (HRegionInfo info: regions) { - master.getRegionManager().setUnassigned(info, true); + masterStatus.getRegionManager().setUnassigned(info, true); } } private class ScanRootRegion extends RetryableMetaOperation { - ScanRootRegion(MetaRegion m, HMaster master) { - super(m, master); + ScanRootRegion(MetaRegion m, MasterStatus masterStatus) { + super(m, masterStatus); } public Boolean call() throws IOException { if (LOG.isDebugEnabled()) { LOG.debug("Process server shutdown scanning root region on " + - master.getRegionManager().getRootRegionLocation().getBindAddress()); + masterStatus.getRegionManager().getRootRegionLocation().getBindAddress()); } Scan scan = new Scan(); scan.addFamily(HConstants.CATALOG_FAMILY); @@ -264,8 +264,8 @@ } private class ScanMetaRegions extends RetryableMetaOperation { - ScanMetaRegions(MetaRegion m, HMaster master) { - super(m, master); + ScanMetaRegions(MetaRegion m, MasterStatus masterStatus) { + super(m, masterStatus); } public Boolean call() throws IOException { @@ -286,21 +286,21 @@ protected boolean process() throws IOException { LOG.info("Process shutdown of server " + this.deadServer + ": logSplit: " + logSplit + ", rootRescanned: " + rootRescanned + - ", numberOfMetaRegions: " + master.getRegionManager().numMetaRegions() + + ", numberOfMetaRegions: " + masterStatus.getRegionManager().numMetaRegions() + ", onlineMetaRegions.size(): " + - master.getRegionManager().numOnlineMetaRegions()); + masterStatus.getRegionManager().numOnlineMetaRegions()); if (!logSplit) { // Process the old log file - if (this.master.getFileSystem().exists(rsLogDir)) { - if (!master.splitLogLock.tryLock()) { + if (this.masterStatus.getFileSystem().exists(rsLogDir)) { + if (!masterStatus.getSplitLogLock().tryLock()) { return false; } try { - HLog.splitLog(master.getRootDir(), rsLogDir, - this.master.getOldLogDir(), this.master.getFileSystem(), - this.master.getConfiguration()); + HLog.splitLog(masterStatus.getRootDir(), rsLogDir, + this.masterStatus.getOldLogDir(), this.masterStatus.getFileSystem(), + this.masterStatus.getConfiguration()); } finally { - master.splitLogLock.unlock(); + masterStatus.getSplitLogLock().unlock(); } } logSplit = true; @@ -308,13 +308,13 @@ LOG.info("Log split complete, meta reassignment and scanning:"); if (this.isRootServer) { LOG.info("ProcessServerShutdown reassigning ROOT region"); - master.getRegionManager().reassignRootRegion(); + masterStatus.getRegionManager().reassignRootRegion(); isRootServer = false; // prevent double reassignment... heh. } for (MetaRegion metaRegion : metaRegions) { LOG.info("ProcessServerShutdown setting to unassigned: " + metaRegion.toString()); - master.getRegionManager().setUnassigned(metaRegion.getRegionInfo(), true); + masterStatus.getRegionManager().setUnassigned(metaRegion.getRegionInfo(), true); } // one the meta regions are online, "forget" about them. Since there are explicit // checks below to make sure meta/root are online, this is likely to occur. @@ -330,8 +330,8 @@ if (!rootRescanned) { // Scan the ROOT region Boolean result = new ScanRootRegion( - new MetaRegion(master.getRegionManager().getRootRegionLocation(), - HRegionInfo.ROOT_REGIONINFO), this.master).doWithRetries(); + new MetaRegion(masterStatus.getRegionManager().getRootRegionLocation(), + HRegionInfo.ROOT_REGIONINFO), this.masterStatus).doWithRetries(); if (result == null) { // Master is closing - give up return true; @@ -339,7 +339,7 @@ if (LOG.isDebugEnabled()) { LOG.debug("Process server shutdown scanning root region on " + - master.getRegionManager().getRootRegionLocation().getBindAddress() + + masterStatus.getRegionManager().getRootRegionLocation().getBindAddress() + " finished " + Thread.currentThread().getName()); } rootRescanned = true; @@ -352,9 +352,9 @@ return true; } - List regions = master.getRegionManager().getListOfOnlineMetaRegions(); + List regions = masterStatus.getRegionManager().getListOfOnlineMetaRegions(); for (MetaRegion r: regions) { - Boolean result = new ScanMetaRegions(r, this.master).doWithRetries(); + Boolean result = new ScanMetaRegions(r, this.masterStatus).doWithRetries(); if (result == null) { break; } @@ -365,7 +365,7 @@ } closeRegionsInTransition(); - this.master.getServerManager().removeDeadServer(deadServer); + this.masterStatus.getServerManager().removeDeadServer(deadServer); if (LOG.isDebugEnabled()) { LOG.debug("Removed " + deadServer + " from deadservers Map"); } Index: src/main/java/org/apache/hadoop/hbase/master/HMaster.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/HMaster.java (revision 957672) +++ src/main/java/org/apache/hadoop/hbase/master/HMaster.java (working copy) @@ -62,13 +62,12 @@ import org.apache.hadoop.hbase.TableExistsException; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.HBaseAdmin; +import org.apache.hadoop.hbase.client.MetaScanner; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.MetaScanner; -import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.ServerConnection; import org.apache.hadoop.hbase.client.ServerConnectionManager; -import org.apache.hadoop.hbase.executor.HBaseEventHandler; +import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor; import org.apache.hadoop.hbase.executor.HBaseExecutorService; import org.apache.hadoop.hbase.executor.HBaseEventHandler.HBaseEventType; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; @@ -113,7 +112,7 @@ * @see Watcher */ public class HMaster extends Thread implements HMasterInterface, - HMasterRegionInterface, Watcher { + HMasterRegionInterface, Watcher, MasterStatus { // MASTER is name of the webapp and the attribute name used stuffing this //instance into web context. public static final String MASTER = "master"; @@ -354,7 +353,7 @@ * If not, sets closed * @return false if file system is not available */ - protected boolean checkFileSystem() { + public boolean checkFileSystem() { if (this.fsOk) { try { FSUtils.checkFileSystemAvailable(this.fs); @@ -415,27 +414,27 @@ return this.regionManager; } - int getThreadWakeFrequency() { + public int getThreadWakeFrequency() { return this.threadWakeFrequency; } - FileSystem getFileSystem() { + public FileSystem getFileSystem() { return this.fs; } - AtomicBoolean getShutdownRequested() { + public AtomicBoolean getShutdownRequested() { return this.shutdownRequested; } - AtomicBoolean getClosed() { + public AtomicBoolean getClosed() { return this.closed; } - boolean isClosed() { + public boolean isClosed() { return this.closed.get(); } - ServerConnection getServerConnection() { + public ServerConnection getServerConnection() { return this.connection; } @@ -448,11 +447,11 @@ } // These methods are so don't have to pollute RegionManager with ServerManager. - SortedMap> getLoadToServers() { + public SortedMap> getLoadToServers() { return this.serverManager.getLoadToServers(); } - int numServers() { + public int numServers() { return this.serverManager.numServers(); } @@ -460,7 +459,7 @@ return this.serverManager.getAverageLoad(); } - public RegionServerOperationQueue getRegionServerOperationQueue () { + public RegionServerOperationQueue getRegionServerOperationQueue() { return this.regionServerOperationQueue; } @@ -478,7 +477,7 @@ * @param l * @param m */ - void getLightServers(final HServerLoad l, + public void getLightServers(final HServerLoad l, SortedMap> m) { this.serverManager.getLightServers(l, m); } @@ -644,6 +643,10 @@ } } } + + public Lock getSplitLogLock() { + return splitLogLock; + } /* * Start up all services. If any of these threads gets an unhandled exception @@ -686,7 +689,7 @@ /* * Start shutting down the master */ - void startShutdown() { + public void startShutdown() { this.closed.set(true); this.regionManager.stopScanners(); this.regionServerOperationQueue.shutdown(); @@ -1103,7 +1106,7 @@ * @return Null or found HRegionInfo. * @throws IOException */ - HRegionInfo getHRegionInfo(final byte [] row, final Result res) + public HRegionInfo getHRegionInfo(final byte [] row, final Result res) throws IOException { byte[] regioninfo = res.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); @@ -1135,7 +1138,7 @@ * @param metaRegionName name of the meta region we scanned * @param emptyRows the row keys that had empty HRegionInfos */ - protected void deleteEmptyMetaRows(HRegionInterface s, + public void deleteEmptyMetaRows(HRegionInterface s, byte [] metaRegionName, List emptyRows) { for (byte [] regionName: emptyRows) { Index: src/main/java/org/apache/hadoop/hbase/master/ProcessRegionClose.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/ProcessRegionClose.java (revision 957672) +++ src/main/java/org/apache/hadoop/hbase/master/ProcessRegionClose.java (working copy) @@ -37,15 +37,15 @@ protected final boolean reassignRegion; /** - * @param master + * @param masterStatus * @param regionInfo Region to operate on * @param offlineRegion if true, set the region to offline in meta * @param reassignRegion if true, region is to be reassigned */ - public ProcessRegionClose(HMaster master, HRegionInfo regionInfo, + public ProcessRegionClose(MasterStatus masterStatus, HRegionInfo regionInfo, boolean offlineRegion, boolean reassignRegion) { - super(master, regionInfo); + super(masterStatus, regionInfo); this.offlineRegion = offlineRegion; this.reassignRegion = reassignRegion; } @@ -68,7 +68,7 @@ Boolean result = null; if (offlineRegion || reassignRegion) { result = - new RetryableMetaOperation(getMetaRegion(), this.master) { + new RetryableMetaOperation(getMetaRegion(), this.masterStatus) { public Boolean call() throws IOException { @@ -83,14 +83,14 @@ // set of regions in transition HRegion.offlineRegionInMETA(server, metaRegionName, regionInfo); - master.getRegionManager().removeRegion(regionInfo); + masterStatus.getRegionManager().removeRegion(regionInfo); LOG.info("region closed: " + regionInfo.getRegionNameAsString()); } else { // we are reassigning the region eventually, so set it unassigned // and remove the server info HRegion.cleanRegionInMETA(server, metaRegionName, regionInfo); - master.getRegionManager().setUnassigned(regionInfo, false); + masterStatus.getRegionManager().setUnassigned(regionInfo, false); LOG.info("region set as unassigned: " + regionInfo.getRegionNameAsString()); } } Index: src/main/java/org/apache/hadoop/hbase/master/RegionManager.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/RegionManager.java (revision 957672) +++ src/main/java/org/apache/hadoop/hbase/master/RegionManager.java (working copy) @@ -106,7 +106,7 @@ // How many regions to assign a server at a time. private final int maxAssignInOneGo; - final HMaster master; + final MasterStatus masterStatus; private final LoadBalancer loadBalancer; /** Set of regions to split. */ @@ -132,20 +132,20 @@ private final int zooKeeperNumRetries; private final int zooKeeperPause; - RegionManager(HMaster master) throws IOException { - Configuration conf = master.getConfiguration(); + RegionManager(MasterStatus masterStatus) throws IOException { + Configuration conf = masterStatus.getConfiguration(); - this.master = master; + this.masterStatus = masterStatus; this.zkWrapper = ZooKeeperWrapper.getInstance(conf, HMaster.class.getName()); this.maxAssignInOneGo = conf.getInt("hbase.regions.percheckin", 10); this.loadBalancer = new LoadBalancer(conf); // The root region - rootScannerThread = new RootScanner(master); + rootScannerThread = new RootScanner(masterStatus); // Scans the meta table - metaScannerThread = new MetaScanner(master); + metaScannerThread = new MetaScanner(masterStatus); zooKeeperNumRetries = conf.getInt(HConstants.ZOOKEEPER_RETRIES, HConstants.DEFAULT_ZOOKEEPER_RETRIES); @@ -173,7 +173,7 @@ void reassignRootRegion() { unsetRootRegion(); - if (!master.getShutdownRequested().get()) { + if (!masterStatus.getShutdownRequested().get()) { synchronized (regionsInTransition) { String regionName = HRegionInfo.ROOT_REGIONINFO.getRegionNameAsString(); byte[] data = null; @@ -203,7 +203,7 @@ void assignRegions(HServerInfo info, HRegionInfo[] mostLoadedRegions, ArrayList returnMsgs) { HServerLoad thisServersLoad = info.getLoad(); - boolean isSingleServer = this.master.numServers() == 1; + boolean isSingleServer = this.masterStatus.numServers() == 1; // figure out what regions need to be assigned and aren't currently being // worked on elsewhere. @@ -277,7 +277,7 @@ // No other servers with same load. // Split regions over all available servers nregions = (int) Math.ceil((1.0 * nRegionsToAssign)/ - (1.0 * master.getServerManager().numServers())); + (1.0 * masterStatus.getServerManager().numServers())); } } else { // Assign all regions to this server @@ -364,7 +364,7 @@ final HServerLoad thisServersLoad) { SortedMap> lightServers = new TreeMap>(); - this.master.getLightServers(thisServersLoad, lightServers); + this.masterStatus.getLightServers(thisServersLoad, lightServers); // Examine the list of servers that are more lightly loaded than this one. // Pretend that we will assign regions to these more lightly loaded servers // until they reach load equal with ours. Then, see how many regions are left @@ -435,7 +435,7 @@ continue; } if (!i.isMetaRegion() && - !master.getServerManager().canAssignUserRegions()) { + !masterStatus.getServerManager().canAssignUserRegions()) { LOG.debug("user region " + i.getRegionNameAsString() + " is in transition but not enough servers yet"); continue; @@ -457,9 +457,9 @@ SortedMap> heavyServers = new TreeMap>(); - synchronized (master.getLoadToServers()) { + synchronized (masterStatus.getLoadToServers()) { heavyServers.putAll( - master.getLoadToServers().tailMap(referenceLoad)); + masterStatus.getLoadToServers().tailMap(referenceLoad)); } int nservers = 0; for (Map.Entry> e : heavyServers.entrySet()) { @@ -558,12 +558,12 @@ public int countRegionsOnFS() throws IOException { int regions = 0; FileStatus [] tableDirs = - this.master.getFileSystem().listStatus(this.master.getRootDir(), new TableDirFilter()); + this.masterStatus.getFileSystem().listStatus(this.masterStatus.getRootDir(), new TableDirFilter()); FileStatus[] regionDirs; RegionDirFilter rdf = new RegionDirFilter(); for(FileStatus tabledir : tableDirs) { if(tabledir.isDir()) { - regionDirs = this.master.getFileSystem().listStatus(tabledir.getPath(), rdf); + regionDirs = this.masterStatus.getFileSystem().listStatus(tabledir.getPath(), rdf); regions += regionDirs.length; } } @@ -633,8 +633,8 @@ } catch(Exception iex) { LOG.warn("meta scanner", iex); } - master.getZooKeeperWrapper().clearRSDirectory(); - master.getZooKeeperWrapper().close(); + masterStatus.getZooKeeperWrapper().clearRSDirectory(); + masterStatus.getZooKeeperWrapper().close(); } /** @@ -720,7 +720,7 @@ int prefixlen = META_REGION_PREFIX.length; if (row.length > prefixlen && Bytes.compareTo(META_REGION_PREFIX, 0, prefixlen, row, 0, prefixlen) == 0) { - return new MetaRegion(this.master.getRegionManager().getRootRegionLocation(), + return new MetaRegion(this.masterStatus.getRegionManager().getRootRegionLocation(), HRegionInfo.ROOT_REGIONINFO); } return this.onlineMetaRegions.floorEntry(row).getValue(); @@ -740,8 +740,8 @@ byte [] metaRegionName) throws IOException { // 2. Create the HRegion - HRegion region = HRegion.createHRegion(newRegion, this.master.getRootDir(), - master.getConfiguration()); + HRegion region = HRegion.createHRegion(newRegion, this.masterStatus.getRootDir(), + masterStatus.getConfiguration()); // 3. Insert into meta HRegionInfo info = region.getRegionInfo(); @@ -809,8 +809,8 @@ } public boolean isRootServer(HServerAddress server) { - return this.master.getRegionManager().getRootRegionLocation() != null && - server.equals(master.getRegionManager().getRootRegionLocation()); + return this.masterStatus.getRegionManager().getRootRegionLocation() != null && + server.equals(masterStatus.getRegionManager().getRootRegionLocation()); } /** @@ -924,8 +924,8 @@ // check to see if ROOT and/or .META. are on this server, reassign them. // use master.getRootRegionLocation. - if (master.getRegionManager().getRootRegionLocation() != null && - server.equals(master.getRegionManager().getRootRegionLocation())) { + if (masterStatus.getRegionManager().getRootRegionLocation() != null && + server.equals(masterStatus.getRegionManager().getRootRegionLocation())) { LOG.info("Offlined ROOT server: " + server); reassignRootRegion(); hasMeta = true; @@ -1175,14 +1175,14 @@ */ public void waitForRootRegionLocation() { synchronized (rootRegionLocation) { - while (!master.getShutdownRequested().get() && - !master.isClosed() && rootRegionLocation.get() == null) { + while (!masterStatus.getShutdownRequested().get() && + !masterStatus.isClosed() && rootRegionLocation.get() == null) { // rootRegionLocation will be filled in when we get an 'open region' // regionServerReport message from the HRegionServer that has been // allocated the ROOT region below. try { // Cycle rather than hold here in case master is closed meantime. - rootRegionLocation.wait(this.master.getThreadWakeFrequency()); + rootRegionLocation.wait(this.masterStatus.getThreadWakeFrequency()); } catch (InterruptedException e) { // continue } @@ -1223,7 +1223,7 @@ private void writeRootRegionLocationToZooKeeper(HServerAddress address) { for (int attempt = 0; attempt < zooKeeperNumRetries; ++attempt) { - if (master.getZooKeeperWrapper().writeRootRegionLocation(address)) { + if (masterStatus.getZooKeeperWrapper().writeRootRegionLocation(address)) { return; } @@ -1233,7 +1233,7 @@ LOG.error("Failed to write root region location to ZooKeeper after " + zooKeeperNumRetries + " retries, shutting down"); - this.master.shutdown(); + this.masterStatus.shutdown(); } /** @@ -1397,7 +1397,7 @@ void loadBalancing(HServerInfo info, HRegionInfo[] mostLoadedRegions, ArrayList returnMsgs) { HServerLoad servLoad = info.getLoad(); - double avg = master.getAverageLoad(); + double avg = masterStatus.getAverageLoad(); // nothing to balance if server load not more then average load if(servLoad.getLoad() <= Math.ceil(avg) || avg <= 2.0) { @@ -1452,7 +1452,7 @@ double avgLoad) { SortedMap> loadToServers = - master.getLoadToServers(); + masterStatus.getLoadToServers(); // check if server most loaded if (!loadToServers.get(loadToServers.lastKey()).contains(srvName)) return 0; Index: src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java (revision 957672) +++ src/main/java/org/apache/hadoop/hbase/master/BaseScanner.java (working copy) @@ -119,7 +119,7 @@ } } private final boolean rootRegion; - protected final HMaster master; + protected final MasterStatus masterStatus; protected boolean initialScanComplete; @@ -130,13 +130,13 @@ // mid-scan final Object scannerLock = new Object(); - BaseScanner(final HMaster master, final boolean rootRegion, + BaseScanner(final MasterStatus masterStatus, final boolean rootRegion, final AtomicBoolean stop) { super("Scanner for " + (rootRegion ? "-ROOT-":".META.") + " table", - master.getConfiguration(). + masterStatus.getConfiguration(). getInt("hbase.master.meta.thread.rescanfrequency", 60 * 1000), stop); this.rootRegion = rootRegion; - this.master = master; + this.masterStatus = masterStatus; this.initialScanComplete = false; } @@ -173,7 +173,7 @@ int rows = 0; try { regionServer = - this.master.getServerConnection().getHRegionConnection(region.getServer()); + this.masterStatus.getServerConnection().getHRegionConnection(region.getServer()); Scan s = new Scan().addFamily(HConstants.CATALOG_FAMILY); // Make this scan do a row at a time otherwise, data can be stale. s.setCaching(1); @@ -183,7 +183,7 @@ if (values == null || values.size() == 0) { break; } - HRegionInfo info = master.getHRegionInfo(values.getRow(), values); + HRegionInfo info = masterStatus.getHRegionInfo(values.getRow(), values); if (info == null) { emptyRows.add(values.getRow()); continue; @@ -199,7 +199,7 @@ rows += 1; } if (rootRegion) { - this.master.getRegionManager().setNumMetaRegions(rows); + this.masterStatus.getRegionManager().setNumMetaRegions(rows); } } catch (IOException e) { if (e instanceof RemoteException) { @@ -229,7 +229,7 @@ if (emptyRows.size() > 0) { LOG.warn("Found " + emptyRows.size() + " rows with empty HRegionInfo " + "while scanning meta region " + Bytes.toString(region.getRegionName())); - this.master.deleteEmptyMetaRows(regionServer, region.getRegionName(), + this.masterStatus.deleteEmptyMetaRows(regionServer, region.getRegionName(), emptyRows); } @@ -309,8 +309,8 @@ LOG.info("Deleting region " + parent.getRegionNameAsString() + " (encoded=" + parent.getEncodedName() + ") because daughter splits no longer hold references"); - HRegion.deleteRegion(this.master.getFileSystem(), - this.master.getRootDir(), parent); + HRegion.deleteRegion(this.masterStatus.getFileSystem(), + this.masterStatus.getRootDir(), parent); HRegion.removeRegionFromMETA(srvr, metaRegionName, parent.getRegionName()); result = true; @@ -499,15 +499,15 @@ return result; } Path tabledir = - new Path(this.master.getRootDir(), split.getTableDesc().getNameAsString()); + new Path(this.masterStatus.getRootDir(), split.getTableDesc().getNameAsString()); for (HColumnDescriptor family: split.getTableDesc().getFamilies()) { Path p = Store.getStoreHomedir(tabledir, split.getEncodedName(), family.getName()); - if (!this.master.getFileSystem().exists(p)) continue; + if (!this.masterStatus.getFileSystem().exists(p)) continue; // Look for reference files. Call listStatus with an anonymous // instance of PathFilter. FileStatus [] ps = - this.master.getFileSystem().listStatus(p, new PathFilter () { + this.masterStatus.getFileSystem().listStatus(p, new PathFilter () { public boolean accept(Path path) { return StoreFile.isReference(path); } @@ -557,18 +557,18 @@ serverName = HServerInfo.getServerName(sa, sc); } HServerInfo storedInfo = null; - synchronized (this.master.getRegionManager()) { + synchronized (this.masterStatus.getRegionManager()) { /* We don't assign regions that are offline, in transition or were on * a dead server. Regions that were on a dead server will get reassigned * by ProcessServerShutdown */ if (info.isOffline() || - this.master.getRegionManager().regionIsInTransition(info.getRegionNameAsString()) || - (serverName != null && this.master.getServerManager().isDead(serverName))) { + this.masterStatus.getRegionManager().regionIsInTransition(info.getRegionNameAsString()) || + (serverName != null && this.masterStatus.getServerManager().isDead(serverName))) { return; } if (serverName != null) { - storedInfo = this.master.getServerManager().getServerInfo(serverName); + storedInfo = this.masterStatus.getServerManager().getServerInfo(serverName); } // If we can't find the HServerInfo, then add it to the list of @@ -581,7 +581,7 @@ ", startCode=" + sc + " unknown."); } // Now get the region assigned - this.master.getRegionManager().setUnassigned(info, true); + this.masterStatus.getRegionManager().setUnassigned(info, true); } } } Index: src/main/java/org/apache/hadoop/hbase/master/ProcessRegionStatusChange.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/ProcessRegionStatusChange.java (revision 957672) +++ src/main/java/org/apache/hadoop/hbase/master/ProcessRegionStatusChange.java (working copy) @@ -33,11 +33,11 @@ protected volatile byte[] metaRegionName = null; /** - * @param master the master + * @param masterStatus the master * @param regionInfo region info */ - public ProcessRegionStatusChange(HMaster master, HRegionInfo regionInfo) { - super(master); + public ProcessRegionStatusChange(MasterStatus masterStatus, HRegionInfo regionInfo) { + super(masterStatus); this.regionInfo = regionInfo; this.isMetaTable = regionInfo.isMetaTable(); } @@ -52,7 +52,7 @@ available = false; } } else { - if (!master.getRegionManager().isInitialRootScanComplete() || + if (!masterStatus.getRegionManager().isInitialRootScanComplete() || !metaTableAvailable()) { // The root region has not been scanned or the meta table is not // available so we can't proceed. @@ -67,11 +67,11 @@ protected MetaRegion getMetaRegion() { if (isMetaTable) { this.metaRegionName = HRegionInfo.ROOT_REGIONINFO.getRegionName(); - this.metaRegion = new MetaRegion(master.getRegionManager().getRootRegionLocation(), + this.metaRegion = new MetaRegion(masterStatus.getRegionManager().getRootRegionLocation(), HRegionInfo.ROOT_REGIONINFO); } else { this.metaRegion = - master.getRegionManager().getFirstMetaRegionForRegion(regionInfo); + masterStatus.getRegionManager().getFirstMetaRegionForRegion(regionInfo); if (this.metaRegion != null) { this.metaRegionName = this.metaRegion.getRegionName(); }