diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon index 497603c..28c2881 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/AssignmentManagerStatusTmpl.jamon @@ -61,8 +61,7 @@ if (toRemove > 0) { Map.Entry e = it.next(); if (HRegionInfo.FIRST_META_REGIONINFO.getEncodedName().equals( e.getKey()) || - HRegionInfo.ROOT_REGIONINFO.getEncodedName().equals( - e.getKey()) || regionIDForOldestRIT.equals(e.getKey())) { + regionIDForOldestRIT.equals(e.getKey())) { // don't remove the meta & the oldest rit regions, they're too interesting! continue; } diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon index 9587af8..bb0277a 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon @@ -20,7 +20,6 @@ limitations under the License. HMaster master; HBaseAdmin admin; Map frags = null; -ServerName rootLocation = null; ServerName metaLocation = null; List servers = null; Set deadServers = null; @@ -154,7 +153,7 @@ org.apache.hadoop.hbase.HBaseConfiguration;
- <%if (rootLocation != null) %> + <%if (metaLocation != null) %> <& catalogTables &>
@@ -254,13 +253,6 @@ org.apache.hadoop.hbase.HBaseConfiguration; Description - - <% Bytes.toString(HConstants.ROOT_TABLE_NAME) %> - <%if (frags != null)%> - <% frags.get("-ROOT-") != null ? frags.get("-ROOT-").intValue() + "%" : "n/a" %> - - The -ROOT- table holds references to all .META. regions. - <%if (metaLocation != null) %> <% Bytes.toString(HConstants.META_TABLE_NAME) %> diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java index bc32198..3963c77 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java @@ -103,7 +103,7 @@ public class HRegionInfo implements Comparable { *

* **NOTE** * - * ROOT, the first META region, and regions created by an older + * The first META region, and regions created by an older * version of HBase (0.20 or prior) will continue to use the * old region name format. */ @@ -143,7 +143,7 @@ public class HRegionInfo implements Comparable { regionName.length - MD5_HEX_LENGTH - 1, MD5_HEX_LENGTH); } else { - // old format region name. ROOT and first META region also + // old format region name. First META region also // use this format.EncodedName is the JenkinsHash value. int hashVal = Math.abs(JenkinsHash.getInstance().hash(regionName, regionName.length, 0)); @@ -155,14 +155,14 @@ public class HRegionInfo implements Comparable { /** * Use logging. * @param encodedRegionName The encoded regionname. - * @return -ROOT- if passed 70236052 or + * @return * .META. if passed 1028785192 else returns * encodedRegionName */ public static String prettyPrint(final String encodedRegionName) { - if (encodedRegionName.equals("70236052")) { + /*if (encodedRegionName.equals("70236052")) { return encodedRegionName + "/-ROOT-"; - } else if (encodedRegionName.equals("1028785192")) { + } else*/ if (encodedRegionName.equals("1028785192")) { return encodedRegionName + "/.META."; } return encodedRegionName; @@ -207,14 +207,14 @@ public class HRegionInfo implements Comparable { /** - * Private constructor used constructing HRegionInfo for the catalog root and + * Private constructor used constructing HRegionInfo for the * first meta regions */ private HRegionInfo(long regionId, byte[] tableName) { super(); this.regionId = regionId; this.tableName = tableName.clone(); - // Note: Root & First Meta regions names are still in old format + // Note: First Meta regions names are still in old format this.regionName = createRegionName(tableName, null, regionId, false); this.regionNameStr = Bytes.toStringBinary(this.regionName); @@ -608,16 +608,10 @@ public class HRegionInfo implements Comparable { } } - /** @return true if this is the root region */ - public boolean isRootRegion() { - return Bytes.equals(tableName, HRegionInfo.ROOT_REGIONINFO.getTableName()); - } - - /** @return true if this region is from a table that is a meta table, - * either .META. or -ROOT- + /** @return true if this region is from .META. */ public boolean isMetaTable() { - return isRootRegion() || isMetaRegion(); + return isMetaRegion(); } /** @return true if this region is a meta region */ @@ -844,7 +838,7 @@ public class HRegionInfo implements Comparable { * @return Comparator to use comparing {@link KeyValue}s. */ public KVComparator getComparator() { - return isRootRegion()? KeyValue.ROOT_COMPARATOR: isMetaRegion()? + return isMetaRegion()? KeyValue.META_COMPARATOR: KeyValue.COMPARATOR; } @@ -888,9 +882,7 @@ public class HRegionInfo implements Comparable { public static HRegionInfo convert(final RegionInfo proto) { if (proto == null) return null; byte [] tableName = proto.getTableName().toByteArray(); - if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) { - return ROOT_REGIONINFO; - } else if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) { + if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) { return FIRST_META_REGIONINFO; } long regionId = proto.getRegionId(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java index 8a383e4..55725f4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java @@ -24,7 +24,6 @@ import java.net.NoRouteToHostException; import java.net.SocketException; import java.net.SocketTimeoutException; import java.net.UnknownHostException; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -43,30 +42,29 @@ import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.zookeeper.MetaNodeTracker; -import org.apache.hadoop.hbase.zookeeper.RootRegionTracker; +import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.ipc.RemoteException; /** - * Tracks the availability of the catalog tables -ROOT- and + * Tracks the availability of the catalog tables * .META.. * * This class is "read-only" in that the locations of the catalog tables cannot * be explicitly set. Instead, ZooKeeper is used to learn of the availability - * and location of -ROOT-. -ROOT- is used to learn of - * the location of .META. If not available in -ROOT-, - * ZooKeeper is used to monitor for a new location of .META.. + * and location of .META.. * *

Call {@link #start()} to start up operation. Call {@link #stop()}} to * interrupt waits and close up shop. */ @InterfaceAudience.Private public class CatalogTracker { + // TODO JDC 11/30 We don't even have ROOT anymore, revisit // TODO: This class needs a rethink. The original intent was that it would be - // the one-stop-shop for root and meta locations and that it would get this + // the one-stop-shop for meta locations and that it would get this // info from reading and watching zk state. The class was to be used by - // servers when they needed to know of root and meta movement but also by - // client-side (inside in HTable) so rather than figure root and meta + // servers when they needed to know of meta movement but also by + // client-side (inside in HTable) so rather than figure meta // locations on fault, the client would instead get notifications out of zk. // // But this original intent is frustrated by the fact that this class has to @@ -109,9 +107,9 @@ public class CatalogTracker { private static final Log LOG = LogFactory.getLog(CatalogTracker.class); private final HConnection connection; private final ZooKeeperWatcher zookeeper; - private final RootRegionTracker rootRegionTracker; - private final MetaNodeTracker metaNodeTracker; - private final AtomicBoolean metaAvailable = new AtomicBoolean(false); + private final MetaRegionTracker metaRegionTracker; + //private final MetaNodeTracker metaNodeTracker; + //private final AtomicBoolean metaAvailable = new AtomicBoolean(false); private boolean instantiatedzkw = false; private Abortable abortable; @@ -123,14 +121,12 @@ public class CatalogTracker { private ServerName metaLocation; /* - * Timeout waiting on root or meta to be set. + * Timeout waiting on meta to be set. */ private final int defaultTimeout; private boolean stopped = false; - static final byte [] ROOT_REGION_NAME = - HRegionInfo.ROOT_REGIONINFO.getRegionName(); static final byte [] META_REGION_NAME = HRegionInfo.FIRST_META_REGIONINFO.getRegionName(); @@ -217,15 +213,16 @@ public class CatalogTracker { } else { this.zookeeper = zk; } - this.rootRegionTracker = new RootRegionTracker(zookeeper, throwableAborter); + this.metaRegionTracker = new MetaRegionTracker(zookeeper, throwableAborter); final CatalogTracker ct = this; + // TODO JDC I don't think this is needed // Override nodeDeleted so we get notified when meta node deleted - this.metaNodeTracker = new MetaNodeTracker(zookeeper, throwableAborter) { + /*this.metaNodeTracker = new MetaNodeTracker(zookeeper, throwableAborter) { public void nodeDeleted(String path) { if (!path.equals(node)) return; - ct.resetMetaLocation(); + this ct.resetMetaLocation(); } - }; + };*/ this.defaultTimeout = defaultTimeout; } @@ -239,12 +236,11 @@ public class CatalogTracker { public void start() throws IOException, InterruptedException { LOG.debug("Starting catalog tracker " + this); try { - this.rootRegionTracker.start(); - this.metaNodeTracker.start(); + this.metaRegionTracker.start(); } catch (RuntimeException e) { Throwable t = e.getCause(); this.abortable.abort(e.getMessage(), t); - throw new IOException("Attempt to start root/meta tracker failed.", t); + throw new IOException("Attempt to start meta tracker failed.", t); } } @@ -256,8 +252,7 @@ public class CatalogTracker { if (!this.stopped) { LOG.debug("Stopping catalog tracker " + this); this.stopped = true; - this.rootRegionTracker.stop(); - this.metaNodeTracker.stop(); + this.metaRegionTracker.stop(); try { if (this.connection != null) { this.connection.close(); @@ -271,60 +266,37 @@ public class CatalogTracker { this.zookeeper.close(); } // Call this and it will interrupt any ongoing waits on meta. - synchronized (this.metaAvailable) { + /*synchronized (this.metaAvailable) { this.metaAvailable.notifyAll(); - } + }*/ } } /** - * Gets the current location for -ROOT- or null if location is + * Gets the current location for .META. or null if location is * not currently available. - * @return {@link ServerName} for server hosting -ROOT- or null - * if none available - * @throws InterruptedException - */ - public ServerName getRootLocation() throws InterruptedException { - return this.rootRegionTracker.getRootRegionLocation(); - } - - /** * @return {@link ServerName} for server hosting .META. or null * if none available + * @throws InterruptedException */ - public ServerName getMetaLocation() { - return this.metaLocation; - } - - /** - * Method used by master on startup trying to figure state of cluster. - * Returns the current meta location unless its null. In this latter case, - * it has not yet been set so go check whats up in -ROOT- and - * return that. - * @return {@link ServerName} for server hosting .META. or if null, - * we'll read the location that is up in -ROOT- table (which - * could be null or just plain stale). - * @throws IOException - */ - public ServerName getMetaLocationOrReadLocationFromRoot() throws IOException { - ServerName sn = getMetaLocation(); - return sn != null? sn: MetaReader.getMetaRegionLocation(this); + public ServerName getMetaLocation() throws InterruptedException { + return this.metaRegionTracker.getMetaRegionLocation(); } /** - * Gets the current location for -ROOT- if available and waits + * Gets the current location for .META. if available and waits * for up to the specified timeout if not immediately available. Returns null * if the timeout elapses before root is available. * @param timeout maximum time to wait for root availability, in milliseconds - * @return {@link ServerName} for server hosting -ROOT- or null + * @return {@link ServerName} for server hosting .META. or null * if none available * @throws InterruptedException if interrupted while waiting - * @throws NotAllMetaRegionsOnlineException if root not available before + * @throws NotAllMetaRegionsOnlineException if meta not available before * timeout */ - public ServerName waitForRoot(final long timeout) + public ServerName waitForMeta(final long timeout) throws InterruptedException, NotAllMetaRegionsOnlineException { - ServerName sn = rootRegionTracker.waitRootRegionLocation(timeout); + ServerName sn = metaRegionTracker.waitMetaRegionLocation(timeout); if (sn == null) { throw new NotAllMetaRegionsOnlineException("Timed out; " + timeout + "ms"); } @@ -332,107 +304,56 @@ public class CatalogTracker { } /** - * Gets a connection to the server hosting root, as reported by ZooKeeper, + * Gets a connection to the server hosting meta, as reported by ZooKeeper, * waiting up to the specified timeout for availability. - * @param timeout How long to wait on root location - * @see #waitForRoot(long) for additional information - * @return connection to server hosting root + * @param timeout How long to wait on meta location + * @see #waitForMeta for additional information + * @return connection to server hosting meta * @throws InterruptedException * @throws NotAllMetaRegionsOnlineException if timed out waiting * @throws IOException - * @deprecated Use #getRootServerConnection(long) + * @deprecated Use #getMetaServerConnection(long) */ - public AdminProtocol waitForRootServerConnection(long timeout) + public AdminProtocol waitForMetaServerConnection(long timeout) throws InterruptedException, NotAllMetaRegionsOnlineException, IOException { - return getRootServerConnection(timeout); + return getMetaServerConnection(timeout); } /** - * Gets a connection to the server hosting root, as reported by ZooKeeper, + * Gets a connection to the server hosting meta, as reported by ZooKeeper, * waiting up to the specified timeout for availability. *

WARNING: Does not retry. Use an {@link HTable} instead. - * @param timeout How long to wait on root location - * @see #waitForRoot(long) for additional information - * @return connection to server hosting root + * @param timeout How long to wait on meta location + * @see #waitForMeta for additional information + * @return connection to server hosting meta * @throws InterruptedException * @throws NotAllMetaRegionsOnlineException if timed out waiting * @throws IOException */ - AdminProtocol getRootServerConnection(long timeout) + AdminProtocol getMetaServerConnection(long timeout) throws InterruptedException, NotAllMetaRegionsOnlineException, IOException { - return getCachedConnection(waitForRoot(timeout)); + return getCachedConnection(waitForMeta(timeout)); } /** - * Gets a connection to the server hosting root, as reported by ZooKeeper, + * Gets a connection to the server hosting meta, as reported by ZooKeeper, * waiting for the default timeout specified on instantiation. - * @see #waitForRoot(long) for additional information - * @return connection to server hosting root + * @see #waitForMeta for additional information + * @return connection to server hosting meta * @throws NotAllMetaRegionsOnlineException if timed out waiting * @throws IOException - * @deprecated Use #getRootServerConnection(long) + * @deprecated Use #getMetaServerConnection(long) */ - public AdminProtocol waitForRootServerConnectionDefault() + public AdminProtocol waitForMetaServerConnectionDefault() throws NotAllMetaRegionsOnlineException, IOException { try { - return getRootServerConnection(this.defaultTimeout); + return getMetaServerConnection(this.defaultTimeout); } catch (InterruptedException e) { throw new NotAllMetaRegionsOnlineException("Interrupted"); } } /** - * Gets a connection to the server currently hosting .META. or - * null if location is not currently available. - *

- * If a location is known, a connection to the cached location is returned. - * If refresh is true, the cached connection is verified first before - * returning. If the connection is not valid, it is reset and rechecked. - *

- * If no location for meta is currently known, method checks ROOT for a new - * location, verifies META is currently there, and returns a cached connection - * to the server hosting META. - * - * @return connection to server hosting meta, null if location not available - * @throws IOException - * @throws InterruptedException - */ - private AdminProtocol getMetaServerConnection() - throws IOException, InterruptedException { - synchronized (metaAvailable) { - if (metaAvailable.get()) { - AdminProtocol current = getCachedConnection(this.metaLocation); - // If we are to refresh, verify we have a good connection by making - // an invocation on it. - if (verifyRegionLocation(current, this.metaLocation, META_REGION_NAME)) { - return current; - } - resetMetaLocation(); - } - // We got here because there is no meta available or because whats - // available is bad. - - // Now read the current .META. content from -ROOT-. Note: This goes via - // an HConnection. It has its own way of figuring root and meta locations - // which we have to wait on. - ServerName newLocation = MetaReader.getMetaRegionLocation(this); - if (newLocation == null) return null; - - AdminProtocol newConnection = getCachedConnection(newLocation); - if (verifyRegionLocation(newConnection, newLocation, META_REGION_NAME)) { - setMetaLocation(newLocation); - return newConnection; - } else { - if (LOG.isTraceEnabled()) { - LOG.trace("New .META. server: " + newLocation + " isn't valid." + - " Cached .META. server: " + this.metaLocation); - } - } - return null; - } - } - - /** * Waits indefinitely for availability of .META.. Used during * cluster startup. Does not verify meta, just that something has been * set up in zk. @@ -455,98 +376,6 @@ public class CatalogTracker { } /** - * Gets the current location for .META. if available and waits - * for up to the specified timeout if not immediately available. Throws an - * exception if timed out waiting. This method differs from {@link #waitForMeta()} - * in that it will go ahead and verify the location gotten from ZooKeeper and - * -ROOT- region by trying to use returned connection. - * @param timeout maximum time to wait for meta availability, in milliseconds - * @return {@link ServerName} for server hosting .META. or null - * if none available - * @throws InterruptedException if interrupted while waiting - * @throws IOException unexpected exception connecting to meta server - * @throws NotAllMetaRegionsOnlineException if meta not available before - * timeout - */ - public ServerName waitForMeta(long timeout) - throws InterruptedException, IOException, NotAllMetaRegionsOnlineException { - long stop = System.currentTimeMillis() + timeout; - long waitTime = Math.min(50, timeout); - synchronized (metaAvailable) { - while(!stopped && (timeout == 0 || System.currentTimeMillis() < stop)) { - if (getMetaServerConnection() != null) { - return metaLocation; - } - // perhaps -ROOT- region isn't available, let us wait a bit and retry. - metaAvailable.wait(waitTime); - } - if (getMetaServerConnection() == null) { - throw new NotAllMetaRegionsOnlineException("Timed out (" + timeout + "ms)"); - } - return metaLocation; - } - } - - /** - * Gets a connection to the server hosting meta, as reported by ZooKeeper, - * waiting up to the specified timeout for availability. - * @see #waitForMeta(long) for additional information - * @return connection to server hosting meta - * @throws InterruptedException - * @throws NotAllMetaRegionsOnlineException if timed out waiting - * @throws IOException - * @deprecated Does not retry; use an HTable instance instead. - */ - public AdminProtocol waitForMetaServerConnection(long timeout) - throws InterruptedException, NotAllMetaRegionsOnlineException, IOException { - return getCachedConnection(waitForMeta(timeout)); - } - - /** - * Gets a connection to the server hosting meta, as reported by ZooKeeper, - * waiting up to the specified timeout for availability. - * Used in tests. - * @see #waitForMeta(long) for additional information - * @return connection to server hosting meta - * @throws NotAllMetaRegionsOnlineException if timed out or interrupted - * @throws IOException - * @deprecated Does not retry; use an HTable instance instead. - */ - public AdminProtocol waitForMetaServerConnectionDefault() - throws NotAllMetaRegionsOnlineException, IOException { - try { - return getCachedConnection(waitForMeta(defaultTimeout)); - } catch (InterruptedException e) { - throw new NotAllMetaRegionsOnlineException("Interrupted"); - } - } - - /** - * Called when we figure current meta is off (called from zk callback). - */ - public void resetMetaLocation() { - LOG.debug("Current cached META location, " + metaLocation + - ", is not valid, resetting"); - synchronized(this.metaAvailable) { - this.metaAvailable.set(false); - this.metaAvailable.notifyAll(); - } - } - - /** - * @param metaLocation - */ - void setMetaLocation(final ServerName metaLocation) { - LOG.debug("Set new cached META location: " + metaLocation); - synchronized (this.metaAvailable) { - this.metaLocation = metaLocation; - this.metaAvailable.set(true); - // no synchronization because these are private and already under lock - this.metaAvailable.notifyAll(); - } - } - - /** * @param sn ServerName to get a connection against. * @return The AdminProtocol we got when we connected to sn * May have come from cache, may not be good, may have been setup by this @@ -596,7 +425,7 @@ public class CatalogTracker { * Verify we can connect to hostingServer and that its carrying * regionName. * @param hostingServer Interface to the server hosting regionName - * @param serverName The servername that goes with the metaServer + * @param address The servername that goes with the metaServer * Interface. Used logging. * @param regionName The regionname we are interested in. * @return True if we were able to verify the region located at other side of @@ -642,18 +471,18 @@ public class CatalogTracker { } /** - * Verify -ROOT- is deployed and accessible. - * @param timeout How long to wait on zk for root address (passed through to - * the internal call to {@link #waitForRootServerConnection(long)}. - * @return True if the -ROOT- location is healthy. + * Verify .META. is deployed and accessible. + * @param timeout How long to wait on zk for meta address (passed through to + * the internal call to {@link #waitForMetaServerConnection(long)}. + * @return True if the .META. location is healthy. * @throws IOException * @throws InterruptedException */ - public boolean verifyRootRegionLocation(final long timeout) + public boolean verifyMetaRegionLocation(final long timeout) throws InterruptedException, IOException { AdminProtocol connection = null; try { - connection = waitForRootServerConnection(timeout); + connection = waitForMetaServerConnection(timeout); } catch (NotAllMetaRegionsOnlineException e) { // Pass } catch (ServerNotRunningYetException e) { @@ -663,39 +492,13 @@ public class CatalogTracker { } return (connection == null)? false: verifyRegionLocation(connection, - this.rootRegionTracker.getRootRegionLocation(), ROOT_REGION_NAME); - } - - /** - * Verify .META. is deployed and accessible. - * @param timeout How long to wait on zk for .META. address - * (passed through to the internal call to {@link #waitForMetaServerConnection(long)}. - * @return True if the .META. location is healthy. - * @throws IOException Some unexpected IOE. - * @throws InterruptedException - */ - public boolean verifyMetaRegionLocation(final long timeout) - throws InterruptedException, IOException { - AdminProtocol connection = null; - try { - connection = waitForMetaServerConnection(timeout); - } catch (NotAllMetaRegionsOnlineException e) { - // Pass - } catch (ServerNotRunningYetException e) { - // Pass -- remote server is not up so can't be carrying .META. - } catch (UnknownHostException e) { - // Pass -- server name doesn't resolve so it can't be assigned anything. - } catch (RetriesExhaustedException e) { - // Pass -- failed after bunch of retries. - LOG.debug("Failed verify meta region location after retries", e); - } - return connection != null; + this.metaRegionTracker.getMetaRegionLocation(), META_REGION_NAME); } // Used by tests. - MetaNodeTracker getMetaNodeTracker() { + /*MetaNodeTracker getMetaNodeTracker() { return this.metaNodeTracker; - } + }*/ public HConnection getConnection() { return this.connection; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java index 694e09b..5e195bc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java @@ -86,10 +86,10 @@ public class MetaEditor { * @param p Put to add to .META. * @throws IOException */ - static void putToRootTable(final CatalogTracker ct, final Put p) + /*static void putToRootTable(final CatalogTracker ct, final Put p) throws IOException { put(MetaReader.getRootHTable(ct), p); - } + }*/ /** * Put the passed p to a catalog table. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationConvertingToPB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationConvertingToPB.java index 2b8c98e..896e419 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationConvertingToPB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaMigrationConvertingToPB.java @@ -164,7 +164,7 @@ public class MetaMigrationConvertingToPB { throws IOException { LOG.info("Starting update of ROOT"); ConvertToPBMetaVisitor v = new ConvertToPBMetaVisitor(masterServices); - MetaReader.fullScan(masterServices.getCatalogTracker(), v, null, true); + MetaReader.fullScan(masterServices.getCatalogTracker(), v, null); LOG.info("Finished update of ROOT. Total rows updated:" + v.numMigratedRows); return v.numMigratedRows; } @@ -177,7 +177,7 @@ public class MetaMigrationConvertingToPB { LOG.info("Starting update of META"); ConvertToPBMetaVisitor v = new ConvertToPBMetaVisitor(masterServices); MetaReader.fullScan(masterServices.getCatalogTracker(), v); - updateRootWithMetaMigrationStatus(masterServices.getCatalogTracker()); + //updateRootWithMetaMigrationStatus(masterServices.getCatalogTracker()); LOG.info("Finished update of META. Total rows updated:" + v.numMigratedRows); return v.numMigratedRows; } @@ -192,7 +192,8 @@ public class MetaMigrationConvertingToPB { Put p = new Put(HRegionInfo.FIRST_META_REGIONINFO.getRegionName()); p.add(HConstants.CATALOG_FAMILY, HConstants.META_VERSION_QUALIFIER, Bytes.toBytes(HConstants.META_VERSION)); - MetaEditor.putToRootTable(catalogTracker, p); + // TODO so wrong + //MetaEditor.putToRootTable(catalogTracker, p); LOG.info("Updated -ROOT- meta version=" + HConstants.META_VERSION); } @@ -202,7 +203,7 @@ public class MetaMigrationConvertingToPB { * @throws IOException */ static boolean isMetaHRIUpdated(final CatalogTracker catalogTracker) throws IOException { - List results = MetaReader.fullScanOfRoot(catalogTracker); + List results = MetaReader.fullScanOfMeta(catalogTracker); if (results == null || results.isEmpty()) { LOG.info(".META. is not migrated"); return false; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java index ea9da0c..2dcafd1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java @@ -62,21 +62,6 @@ public class MetaReader { } /** - * @param row - * @return True if row is row of -ROOT- table. - */ - private static boolean isRootTableRow(final byte [] row) { - if (row.length < META_REGION_PREFIX.length + 2 /* ',', + '1' */) { - // Can't be meta table region. - return false; - } - // Compare the prefix of row. If it matches META_REGION_PREFIX prefix, - // then this is row from -ROOT_ table. - return Bytes.equals(row, 0, META_REGION_PREFIX.length, - META_REGION_PREFIX, 0, META_REGION_PREFIX.length); - } - - /** * Performs a full scan of .META., skipping regions from any * tables in the specified set of disabled tables. * @param catalogTracker @@ -144,14 +129,14 @@ public class MetaReader { } /** - * Performs a full scan of a -ROOT- table. + * Performs a full scan of a .META. table. * @return List of {@link Result} * @throws IOException */ - public static List fullScanOfRoot(CatalogTracker catalogTracker) + public static List fullScanOfMeta(CatalogTracker catalogTracker) throws IOException { CollectAllVisitor v = new CollectAllVisitor(); - fullScan(catalogTracker, v, null, true); + fullScan(catalogTracker, v, null); return v.getResults(); } @@ -168,21 +153,6 @@ public class MetaReader { } /** - * Performs a full scan of .META.. - * @param catalogTracker - * @param visitor Visitor invoked against each row. - * @param startrow Where to start the scan. Pass null if want to begin scan - * at first row (The visitor will stop the Scan when its done so no need to - * pass a stoprow). - * @throws IOException - */ - public static void fullScan(CatalogTracker catalogTracker, - final Visitor visitor, final byte [] startrow) - throws IOException { - fullScan(catalogTracker, visitor, startrow, false); - } - - /** * Callers should call close on the returned {@link HTable} instance. * @param catalogTracker We'll use this catalogtracker's connection * @param tableName Table to get an {@link HTable} against. @@ -209,9 +179,7 @@ public class MetaReader { static HTable getCatalogHTable(final CatalogTracker catalogTracker, final byte [] row) throws IOException { - return isRootTableRow(row)? - getRootHTable(catalogTracker): - getMetaHTable(catalogTracker); + return getMetaHTable(catalogTracker); } /** @@ -226,17 +194,6 @@ public class MetaReader { } /** - * Callers should call close on the returned {@link HTable} instance. - * @param ct - * @return An {@link HTable} for -ROOT- - * @throws IOException - */ - static HTable getRootHTable(final CatalogTracker ct) - throws IOException { - return getHTable(ct, HConstants.ROOT_TABLE_NAME); - } - - /** * @param t Table to use (will be closed when done). * @param g Get to run * @throws IOException @@ -250,19 +207,6 @@ public class MetaReader { } /** - * Gets the location of .META. region by reading content of - * -ROOT-. - * @param ct - * @return location of .META. region as a {@link ServerName} or - * null if not found - * @throws IOException - */ - static ServerName getMetaRegionLocation(final CatalogTracker ct) - throws IOException { - return MetaReader.readRegionLocation(ct, CatalogTracker.META_REGION_NAME); - } - - /** * Reads the location of the specified region * @param catalogTracker * @param regionName region whose location we are after @@ -303,8 +247,7 @@ public class MetaReader { public static boolean tableExists(CatalogTracker catalogTracker, String tableName) throws IOException { - if (tableName.equals(HTableDescriptor.ROOT_TABLEDESC.getNameAsString()) || - tableName.equals(HTableDescriptor.META_TABLEDESC.getNameAsString())) { + if (tableName.equals(HTableDescriptor.META_TABLEDESC.getNameAsString())) { // Catalog tables always exist. return true; } @@ -452,15 +395,6 @@ public class MetaReader { getTableRegionsAndLocations(final CatalogTracker catalogTracker, final byte [] tableName, final boolean excludeOfflinedSplitParents) throws IOException, InterruptedException { - if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) { - // If root, do a bit of special handling. - ServerName serverName = catalogTracker.getRootLocation(); - List> list = - new ArrayList>(); - list.add(new Pair(HRegionInfo.ROOT_REGIONINFO, - serverName)); - return list; - } // Make a version of CollectingVisitor that collects HRegionInfo and ServerAddress CollectingVisitor> visitor = new CollectingVisitor>() { @@ -488,8 +422,7 @@ public class MetaReader { this.results.add(this.current); } }; - fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName), - Bytes.equals(tableName, HConstants.META_TABLE_NAME)); + fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName)); return visitor.getResults(); } @@ -547,23 +480,21 @@ public class MetaReader { * @param visitor Visitor invoked against each row. * @param startrow Where to start the scan. Pass null if want to begin scan * at first row. - * @param scanRoot True if we are to scan -ROOT- rather than * .META., the default (pass false to scan .META.) * @throws IOException */ - static void fullScan(CatalogTracker catalogTracker, - final Visitor visitor, final byte [] startrow, final boolean scanRoot) + public static void fullScan(CatalogTracker catalogTracker, + final Visitor visitor, final byte [] startrow) throws IOException { Scan scan = new Scan(); if (startrow != null) scan.setStartRow(startrow); - if (startrow == null && !scanRoot) { + if (startrow == null) { int caching = catalogTracker.getConnection().getConfiguration() .getInt(HConstants.HBASE_META_SCANNER_CACHING, 100); scan.setCaching(caching); } scan.addFamily(HConstants.CATALOG_FAMILY); - HTable metaTable = scanRoot? - getRootHTable(catalogTracker): getMetaHTable(catalogTracker); + HTable metaTable = getMetaHTable(catalogTracker); ResultScanner scanner = metaTable.getScanner(scan); try { Result data; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java index 447a64e..2269e8a9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java @@ -88,12 +88,8 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.SoftValueSortedMap; import org.apache.hadoop.hbase.util.Triple; -import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; -import org.apache.hadoop.hbase.zookeeper.RootRegionTracker; -import org.apache.hadoop.hbase.zookeeper.ZKClusterId; -import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly; -import org.apache.hadoop.hbase.zookeeper.ZKUtil; -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.hbase.zookeeper.*; +import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker; import org.apache.hadoop.ipc.RemoteException; import org.apache.zookeeper.KeeperException; @@ -108,7 +104,7 @@ import com.google.protobuf.ServiceException; * instance does not change its object identity). Sharing {@link HConnection} * instances is usually what you want; all clients of the {@link HConnection} * instances share the HConnections' cache of Region locations rather than each - * having to discover for itself the location of meta, root, etc. It makes + * having to discover for itself the location of meta, etc. It makes * sense for the likes of the pool of HTables class {@link HTablePool}, for * instance (If concerned that a single {@link HConnection} is insufficient * for sharing amongst clients in say an heavily-multithreaded environment, @@ -871,10 +867,6 @@ public class HConnectionManager { */ private boolean testTableOnlineState(byte [] tableName, boolean online) throws IOException { - if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) { - // The root region is always enabled - return online; - } String tableNameStr = Bytes.toString(tableName); ZooKeeperKeepAliveConnection zkw = getKeepAliveZooKeeperWatcher(); try { @@ -934,18 +926,18 @@ public class HConnectionManager { "table name cannot be null or zero length"); } - if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) { + if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) { ZooKeeperKeepAliveConnection zkw = getKeepAliveZooKeeperWatcher(); try { - LOG.debug("Looking up root region location in ZK," + + LOG.debug("Looking up meta region location in ZK," + " connection=" + this); ServerName servername = - RootRegionTracker.blockUntilAvailable(zkw, this.rpcTimeout); + MetaRegionTracker.blockUntilAvailable(zkw, this.rpcTimeout); - LOG.debug("Looked up root region location, connection=" + this + + LOG.debug("Looked up meta region location, connection=" + this + "; serverName=" + ((servername == null) ? "null" : servername)); if (servername == null) return null; - return new HRegionLocation(HRegionInfo.ROOT_REGIONINFO, + return new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, servername.getHostname(), servername.getPort()); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -953,9 +945,6 @@ public class HConnectionManager { } finally { zkw.close(); } - } else if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) { - return locateRegionInMeta(HConstants.ROOT_TABLE_NAME, tableName, row, - useCache, metaRegionLock, retry); } else { // Region not in the cache - have to go to the meta RS return locateRegionInMeta(HConstants.META_TABLE_NAME, tableName, row, @@ -1015,7 +1004,7 @@ public class HConnectionManager { } /* - * Search one of the meta tables (-ROOT- or .META.) for the HRegionLocation + * Search one of the meta table for the HRegionLocation * info that contains the table and row we're seeking. */ private HRegionLocation locateRegionInMeta(final byte [] parentTable, @@ -1045,7 +1034,7 @@ public class HConnectionManager { HRegionLocation metaLocation = null; try { - // locate the root or meta region + // locate the meta region metaLocation = locateRegion(parentTable, metaKey, true, false); // If null still, go around again. if (metaLocation == null) continue; @@ -1077,7 +1066,7 @@ public class HConnectionManager { deleteCachedLocation(tableName, row); } - // Query the root or meta region for the location of the meta region + // Query the meta region for the location of the meta region regionInfoRow = ProtobufUtil.getRowOrBefore(server, metaLocation.getRegionInfo().getRegionName(), metaKey, HConstants.CATALOG_FAMILY); @@ -2403,9 +2392,6 @@ public class HConnectionManager { public HTableDescriptor getHTableDescriptor(final byte[] tableName) throws IOException { if (tableName == null || tableName.length == 0) return null; - if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) { - return new UnmodifyableHTableDescriptor(HTableDescriptor.ROOT_TABLEDESC); - } if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) { return HTableDescriptor.META_TABLEDESC; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index 9b5f83f..b7ee713 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -73,7 +73,7 @@ import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException; import org.apache.hadoop.hbase.util.KeyLocker; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Threads; -import org.apache.hadoop.hbase.zookeeper.RootRegionTracker; +import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker; import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.hadoop.hbase.zookeeper.ZKTable; import org.apache.hadoop.hbase.zookeeper.ZKUtil; @@ -464,7 +464,7 @@ public class AssignmentManager extends ZooKeeperListener { } /** - * This call is invoked only (1) master assign root and meta; + * This call is invoked only (1) master assign meta; * (2) during failover mode startup, zk assignment node processing. * The locker is set in the caller. * @@ -520,11 +520,11 @@ public class AssignmentManager extends ZooKeeperListener { case RS_ZK_REGION_OPENING: regionStates.updateRegionState(rt, RegionState.State.OPENING); if (regionInfo.isMetaTable() || !serverManager.isServerOnline(sn)) { - // If ROOT or .META. table is waiting for timeout monitor to assign + // If .META. table is waiting for timeout monitor to assign // it may take lot of time when the assignment.timeout.period is // the default value which may be very long. We will not be able // to serve any request during this time. - // So we will assign the ROOT and .META. region immediately. + // So we will assign the .META. region immediately. // For a user region, if the server is not online, it takes // some time for timeout monitor to kick in. We know the region // won't open. So we will assign the opening @@ -2049,30 +2049,17 @@ public class AssignmentManager extends ZooKeeperListener { } /** - * Assigns the ROOT region. - *

- * Assumes that ROOT is currently closed and is not being actively served by - * any RegionServer. - *

- * Forcibly unsets the current root region location in ZooKeeper and assigns - * ROOT to a random RegionServer. - * @throws KeeperException - */ - public void assignRoot() throws KeeperException { - RootRegionTracker.deleteRootLocation(this.watcher); - assign(HRegionInfo.ROOT_REGIONINFO, true); - } - - /** * Assigns the META region. *

* Assumes that META is currently closed and is not being actively served by * any RegionServer. *

- * Forcibly assigns META to a random RegionServer. + * Forcibly unsets the current meta region location in ZooKeeper and assigns + * META to a random RegionServer. + * @throws KeeperException */ - public void assignMeta() { - // Force assignment to a random server + public void assignMeta() throws KeeperException { + MetaRegionTracker.deleteMetaLocation(this.watcher); assign(HRegionInfo.FIRST_META_REGIONINFO, true); } @@ -2633,10 +2620,6 @@ public class AssignmentManager extends ZooKeeperListener { threadPoolExecutorService.submit(new UnAssignCallable(this, regionInfo)); } - boolean isCarryingRoot(ServerName serverName) { - return isCarryingRegion(serverName, HRegionInfo.ROOT_REGIONINFO); - } - boolean isCarryingMeta(ServerName serverName) { return isCarryingRegion(serverName, HRegionInfo.FIRST_META_REGIONINFO); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 0c428f3..af356f3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -283,7 +283,7 @@ Server { private volatile boolean isActiveMaster = false; // flag set after we complete initialization once active (used for testing) private volatile boolean initialized = false; - // flag set after we complete assignRootAndMeta. + // flag set after we complete assignMeta. private volatile boolean serverShutdownHandlerEnabled = false; // Instance of the hbase executor service. @@ -614,7 +614,7 @@ Server { *

  • Set cluster as UP in ZooKeeper
  • *
  • Wait for RegionServers to check-in
  • *
  • Split logs and perform data recovery, if necessary
  • - *
  • Ensure assignment of root and meta regions
  • + *
  • Ensure assignment of meta regions
  • *
  • Handle either fresh cluster start or master failover
  • * * @@ -689,8 +689,8 @@ Server { status.setStatus("Splitting logs after master startup"); splitLogAfterStartup(this.fileSystemManager); - // Make sure root and meta assigned before proceeding. - if (!assignRootAndMeta(status)) return; + // Make sure meta assigned before proceeding. + if (!assignMeta(status)) return; enableServerShutdownHandler(); // Update meta with new PB serialization if required. i.e migrate all HRI @@ -785,76 +785,42 @@ Server { } /** - * Check -ROOT- and .META. are assigned. If not, + * Check .META. are assigned. If not, * assign them. * @throws InterruptedException * @throws IOException * @throws KeeperException - * @return True if root and meta are healthy, assigned + * @return True if meta is healthy, assigned */ - boolean assignRootAndMeta(MonitoredTask status) + boolean assignMeta(MonitoredTask status) throws InterruptedException, IOException, KeeperException { int assigned = 0; long timeout = this.conf.getLong("hbase.catalog.verification.timeout", 1000); - // Work on ROOT region. Is it in zk in transition? - status.setStatus("Assigning ROOT region"); - assignmentManager.getRegionStates().createRegionState( - HRegionInfo.ROOT_REGIONINFO); - boolean rit = this.assignmentManager. - processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.ROOT_REGIONINFO); - ServerName currentRootServer = null; - boolean rootRegionLocation = catalogTracker.verifyRootRegionLocation(timeout); - if (!rit && !rootRegionLocation) { - currentRootServer = this.catalogTracker.getRootLocation(); - splitLogAndExpireIfOnline(currentRootServer); - this.assignmentManager.assignRoot(); - // Make sure a -ROOT- location is set. - if (!isRootLocation()) return false; - // This guarantees that the transition assigning -ROOT- has completed - this.assignmentManager.waitForAssignment(HRegionInfo.ROOT_REGIONINFO); - assigned++; - } else if (rit && !rootRegionLocation) { - // Make sure a -ROOT- location is set. - if (!isRootLocation()) return false; - // This guarantees that the transition assigning -ROOT- has completed - this.assignmentManager.waitForAssignment(HRegionInfo.ROOT_REGIONINFO); - assigned++; - } else if (rootRegionLocation) { - // Region already assigned. We didn't assign it. Add to in-memory state. - this.assignmentManager.regionOnline(HRegionInfo.ROOT_REGIONINFO, - this.catalogTracker.getRootLocation()); - } - // Enable the ROOT table if on process fail over the RS containing ROOT - // was active. - enableCatalogTables(Bytes.toString(HConstants.ROOT_TABLE_NAME)); - // Check for stopped, just in case - if (this.stopped) return false; - LOG.info("-ROOT- assigned=" + assigned + ", rit=" + rit + - ", location=" + catalogTracker.getRootLocation()); - - // Work on meta region + // Work on .META. region. Is it in zk in transition? status.setStatus("Assigning META region"); assignmentManager.getRegionStates().createRegionState( HRegionInfo.FIRST_META_REGIONINFO); - rit = this.assignmentManager. + boolean rit = this.assignmentManager. processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO); - boolean metaRegionLocation = this.catalogTracker.verifyMetaRegionLocation(timeout); + ServerName currentMetaServer = null; + boolean metaRegionLocation = catalogTracker.verifyMetaRegionLocation(timeout); if (!rit && !metaRegionLocation) { - ServerName currentMetaServer = - this.catalogTracker.getMetaLocationOrReadLocationFromRoot(); - if (currentMetaServer != null - && !currentMetaServer.equals(currentRootServer)) { - splitLogAndExpireIfOnline(currentMetaServer); - } - assignmentManager.assignMeta(); - enableSSHandWaitForMeta(); + currentMetaServer = this.catalogTracker.getMetaLocation(); + splitLogAndExpireIfOnline(currentMetaServer); + this.assignmentManager.assignMeta(); + // Make sure a .META. location is set. + if (!isMetaLocation()) return false; + // This guarantees that the transition assigning .META. has completed + this.assignmentManager.waitForAssignment(HRegionInfo.FIRST_META_REGIONINFO); assigned++; } else if (rit && !metaRegionLocation) { - // Wait until META region added to region server onlineRegions. See HBASE-5875. - enableSSHandWaitForMeta(); + // Make sure a .META. location is set. + if (!isMetaLocation()) return false; + // This guarantees that the transition assigning .META. has completed + this.assignmentManager.waitForAssignment(HRegionInfo.FIRST_META_REGIONINFO); assigned++; - } else { + } else if (metaRegionLocation) { // Region already assigned. We didn't assign it. Add to in-memory state. this.assignmentManager.regionOnline(HRegionInfo.FIRST_META_REGIONINFO, this.catalogTracker.getMetaLocation()); @@ -862,7 +828,7 @@ Server { enableCatalogTables(Bytes.toString(HConstants.META_TABLE_NAME)); LOG.info(".META. assigned=" + assigned + ", rit=" + rit + ", location=" + catalogTracker.getMetaLocation()); - status.setStatus("META and ROOT assigned."); + status.setStatus("META assigned."); return true; } @@ -875,17 +841,17 @@ Server { } /** - * @return True if there a root available + * @return True if there a meta available * @throws InterruptedException */ - private boolean isRootLocation() throws InterruptedException { + private boolean isMetaLocation() throws InterruptedException { // Cycle up here in master rather than down in catalogtracker so we can // check the master stopped flag every so often. while (!this.stopped) { try { - if (this.catalogTracker.waitForRoot(100) != null) break; + if (this.catalogTracker.waitForMeta(100) != null) break; } catch (NotAllMetaRegionsOnlineException e) { - // Ignore. I know -ROOT- is not online yet. + // Ignore. I know .META. is not online yet. } } // We got here because we came of above loop. @@ -1549,8 +1515,7 @@ Server { } private static boolean isCatalogTable(final byte [] tableName) { - return Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME) || - Bytes.equals(tableName, HConstants.META_TABLE_NAME); + return Bytes.equals(tableName, HConstants.META_TABLE_NAME); } @Override @@ -1912,7 +1877,7 @@ Server { * 1. Create a new ZK session. (since our current one is expired) * 2. Try to become a primary master again * 3. Initialize all ZK based system trackers. - * 4. Assign root and meta. (they are already assigned, but we need to update our + * 4. Assign meta. (they are already assigned, but we need to update our * internal memory state to reflect it) * 5. Process any RIT if any during the process of our recovery. * @@ -2122,8 +2087,8 @@ Server { /** * ServerShutdownHandlerEnabled is set false before completing - * assignRootAndMeta to prevent processing of ServerShutdownHandler. - * @return true if assignRootAndMeta has completed; + * assignMeta to prevent processing of ServerShutdownHandler. + * @return true if assignMeta has completed; */ public boolean isServerShutdownHandlerEnabled() { return this.serverShutdownHandlerEnabled; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java index 398b66e..f289c41 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterFileSystem.java @@ -122,8 +122,8 @@ public class MasterFileSystem { /** * Create initial layout in filesystem. *
      - *
    1. Check if the root region exists and is readable, if not create it. - * Create hbase.version and the -ROOT- directory if not one. + *
    2. Check if the meta region exists and is readable, if not create it. + * Create hbase.version and the .META. directory if not one. *
    3. *
    4. Create a log archive directory for RS to put archived logs
    5. *
    @@ -374,13 +374,12 @@ public class MasterFileSystem { } clusterId = FSUtils.getClusterId(fs, rd); - // Make sure the root region directory exists! - if (!FSUtils.rootRegionExists(fs, rd)) { + // Make sure the meta region directory exists! + if (!FSUtils.metaRegionExists(fs, rd)) { bootstrap(rd, c); } - // Create tableinfo-s for ROOT and META if not already there. - FSTableDescriptors.createTableDescriptor(fs, rd, HTableDescriptor.ROOT_TABLEDESC, false); + // Create tableinfo-s for META if not already there. FSTableDescriptors.createTableDescriptor(fs, rd, HTableDescriptor.META_TABLEDESC, false); return rd; @@ -388,25 +387,17 @@ public class MasterFileSystem { private static void bootstrap(final Path rd, final Configuration c) throws IOException { - LOG.info("BOOTSTRAP: creating ROOT and first META regions"); + LOG.info("BOOTSTRAP: creating first META region"); try { // Bootstrapping, make sure blockcache is off. Else, one will be // created here in bootstap and it'll need to be cleaned up. Better to // not make it in first place. Turn off block caching for bootstrap. // Enable after. - HRegionInfo rootHRI = new HRegionInfo(HRegionInfo.ROOT_REGIONINFO); - setInfoFamilyCachingForRoot(false); HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO); setInfoFamilyCachingForMeta(false); - HRegion root = HRegion.createHRegion(rootHRI, rd, c, - HTableDescriptor.ROOT_TABLEDESC); HRegion meta = HRegion.createHRegion(metaHRI, rd, c, HTableDescriptor.META_TABLEDESC); - setInfoFamilyCachingForRoot(true); setInfoFamilyCachingForMeta(true); - // Add first region from the META table to the ROOT region. - HRegion.addRegionToMETA(root, meta); - HRegion.closeHRegion(root); HRegion.closeHRegion(meta); } catch (IOException e) { e = RemoteExceptionHandler.checkIOException(e); @@ -416,19 +407,6 @@ public class MasterFileSystem { } /** - * Enable in-memory caching for -ROOT- - */ - public static void setInfoFamilyCachingForRoot(final boolean b) { - for (HColumnDescriptor hcd: - HTableDescriptor.ROOT_TABLEDESC.getColumnFamilies()) { - if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) { - hcd.setBlockCacheEnabled(b); - hcd.setInMemory(b); - } - } - } - - /** * Enable in memory caching for .META. */ public static void setInfoFamilyCachingForMeta(final boolean b) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java index 79b3802..0fbf011 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterStatusServlet.java @@ -60,8 +60,8 @@ public class MasterStatusServlet extends HttpServlet { Map frags = getFragmentationInfo(master, conf); - ServerName rootLocation = getRootLocationOrNull(master); - ServerName metaLocation = master.getCatalogTracker().getMetaLocation(); + ServerName metaLocation = getMetaLocationOrNull(master); + //ServerName metaLocation = master.getCatalogTracker().getMetaLocation(); List servers = master.getServerManager().getOnlineServersList(); Set deadServers = master.getServerManager().getDeadServers(); @@ -71,7 +71,6 @@ public class MasterStatusServlet extends HttpServlet { tmpl = new MasterStatusTmpl() .setFrags(frags) .setShowAppendWarning(shouldShowAppendWarning(conf)) - .setRootLocation(rootLocation) .setMetaLocation(metaLocation) .setServers(servers) .setDeadServers(deadServers) @@ -88,11 +87,11 @@ public class MasterStatusServlet extends HttpServlet { master, admin); } - private ServerName getRootLocationOrNull(HMaster master) { + private ServerName getMetaLocationOrNull(HMaster master) { try { - return master.getCatalogTracker().getRootLocation(); + return master.getCatalogTracker().getMetaLocation(); } catch (InterruptedException e) { - LOG.warn("Unable to get root location", e); + LOG.warn("Unable to get meta location", e); return null; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java index c811e49..a15a3a8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java @@ -464,7 +464,7 @@ public class RegionStates { } else { for (Map.Entry> e: serverHoldings.entrySet()) { for (HRegionInfo hri: e.getValue()) { - if (hri.isMetaRegion() || hri.isRootRegion()) continue; + if (hri.isMetaRegion()) continue; String tablename = hri.getTableNameAsString(); Map> svrToRegions = result.get(tablename); if (svrToRegions == null) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index 3a5845f..a79590d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -164,7 +164,7 @@ public class ServerManager { * For all the region servers in this set, HLog split is already completed. *

    * ServerShutdownHandler processes a dead server submitted to the handler after - * the handler is enabled. It may not be able to complete the processing because root/meta + * the handler is enabled. It may not be able to complete the processing because meta * is not yet online or master is currently in startup mode. In this case, the dead * server will be parked in this set temporarily. */ @@ -484,28 +484,26 @@ public class ServerManager { return; } - boolean carryingRoot = services.getAssignmentManager().isCarryingRoot(serverName); boolean carryingMeta = services.getAssignmentManager().isCarryingMeta(serverName); - if (carryingRoot || carryingMeta) { + if (carryingMeta) { this.services.getExecutorService().submit(new MetaServerShutdownHandler(this.master, - this.services, this.deadservers, serverName, carryingRoot, carryingMeta)); + this.services, this.deadservers, serverName, carryingMeta)); } else { this.services.getExecutorService().submit(new ServerShutdownHandler(this.master, this.services, this.deadservers, serverName, true)); } LOG.debug("Added=" + serverName + - " to dead servers, submitted shutdown handler to be executed, root=" + - carryingRoot + ", meta=" + carryingMeta); + " to dead servers, submitted shutdown handler to be executed meta=" + carryingMeta); } public synchronized void processDeadServer(final ServerName serverName) { // When assignment manager is cleaning up the zookeeper nodes and rebuilding the - // in-memory region states, region servers could be down. Root/meta table can and + // in-memory region states, region servers could be down. Meta table can and // should be re-assigned, log splitting can be done too. However, it is better to // wait till the cleanup is done before re-assigning user regions. // // We should not wait in the server shutdown handler thread since it can clog - // the handler threads and root/meta table could not be re-assigned in case + // the handler threads and meta table could not be re-assigned in case // the corresponding server is down. So we queue them up here instead. if (!services.getAssignmentManager().isFailoverCleanupDone()) { requeuedDeadServers.add(serverName); @@ -519,7 +517,7 @@ public class ServerManager { /** * Process the servers which died during master's initialization. It will be - * called after HMaster#assignRootAndMeta and AssignmentManager#joinCluster. + * called after HMaster#assignMeta and AssignmentManager#joinCluster. * */ synchronized void processQueuedDeadServers() { if (!services.isServerShutdownHandlerEnabled()) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index d43fd7f..3bf4bd7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -376,7 +376,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { // If this is a special region we always try not to move it. // so clear out r. try again - if (r.isMetaRegion() || r.isRootRegion() ) { + if (r.isMetaRegion()) { r = null; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java index c2be80a..3e0819d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ClosedRegionHandler.java @@ -43,9 +43,8 @@ public class ClosedRegionHandler extends EventHandler implements TotesHRegionInf private final ClosedPriority priority; private enum ClosedPriority { - ROOT (1), - META (2), - USER (3); + META (1), + USER (2); private final int value; ClosedPriority(int value) { @@ -61,9 +60,7 @@ public class ClosedRegionHandler extends EventHandler implements TotesHRegionInf super(server, EventType.RS_ZK_REGION_CLOSED); this.assignmentManager = assignmentManager; this.regionInfo = regionInfo; - if(regionInfo.isRootRegion()) { - priority = ClosedPriority.ROOT; - } else if(regionInfo.isMetaRegion()) { + if(regionInfo.isMetaRegion()) { priority = ClosedPriority.META; } else { priority = ClosedPriority.USER; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java index eb8ec22..55f3855 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java @@ -30,25 +30,19 @@ import org.apache.hadoop.hbase.master.MasterServices; */ @InterfaceAudience.Private public class MetaServerShutdownHandler extends ServerShutdownHandler { - private final boolean carryingRoot; + //private final boolean carryingRoot; private final boolean carryingMeta; public MetaServerShutdownHandler(final Server server, final MasterServices services, final DeadServer deadServers, final ServerName serverName, - final boolean carryingRoot, final boolean carryingMeta) { + final boolean carryingMeta) { super(server, services, deadServers, serverName, EventType.M_META_SERVER_SHUTDOWN, true); - this.carryingRoot = carryingRoot; this.carryingMeta = carryingMeta; } @Override - boolean isCarryingRoot() { - return this.carryingRoot; - } - - @Override boolean isCarryingMeta() { return this.carryingMeta; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/OpenedRegionHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/OpenedRegionHandler.java index 475cacc..f75d424 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/OpenedRegionHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/OpenedRegionHandler.java @@ -44,9 +44,8 @@ public class OpenedRegionHandler extends EventHandler implements TotesHRegionInf private final int expectedVersion; private enum OpenedPriority { - ROOT (1), - META (2), - USER (3); + META (1), + USER (2); private final int value; OpenedPriority(int value) { @@ -65,9 +64,7 @@ public class OpenedRegionHandler extends EventHandler implements TotesHRegionInf this.regionInfo = regionInfo; this.sn = sn; this.expectedVersion = expectedVersion; - if(regionInfo.isRootRegion()) { - priority = OpenedPriority.ROOT; - } else if(regionInfo.isMetaRegion()) { + if(regionInfo.isMetaRegion()) { priority = OpenedPriority.META; } else { priority = OpenedPriority.USER; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java index 53c0182..8bca39b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java @@ -91,22 +91,22 @@ public class ServerShutdownHandler extends EventHandler { } /** - * Before assign the ROOT region, ensure it haven't + * Before assign the META region, ensure it haven't * been assigned by other place *

    - * Under some scenarios, the ROOT region can be opened twice, so it seemed online + * Under some scenarios, the META region can be opened twice, so it seemed online * in two regionserver at the same time. - * If the ROOT region has been assigned, so the operation can be canceled. + * If the META region has been assigned, so the operation can be canceled. * @throws InterruptedException * @throws IOException * @throws KeeperException */ - private void verifyAndAssignRoot() + private void verifyAndAssignMeta() throws InterruptedException, IOException, KeeperException { long timeout = this.server.getConfiguration(). getLong("hbase.catalog.verification.timeout", 1000); - if (!this.server.getCatalogTracker().verifyRootRegionLocation(timeout)) { - this.services.getAssignmentManager().assignRoot(); + if (!this.server.getCatalogTracker().verifyMetaRegionLocation(timeout)) { + this.services.getAssignmentManager().assignMeta(); } } @@ -114,7 +114,7 @@ public class ServerShutdownHandler extends EventHandler { * Failed many times, shutdown processing * @throws IOException */ - private void verifyAndAssignRootWithRetries() throws IOException { + private void verifyAndAssignMetaWithRetries() throws IOException { int iTimes = this.server.getConfiguration().getInt( "hbase.catalog.verification.retries", 10); @@ -124,14 +124,14 @@ public class ServerShutdownHandler extends EventHandler { int iFlag = 0; while (true) { try { - verifyAndAssignRoot(); + verifyAndAssignMeta(); break; } catch (KeeperException e) { - this.server.abort("In server shutdown processing, assigning root", e); + this.server.abort("In server shutdown processing, assigning meta", e); throw new IOException("Aborting", e); } catch (Exception e) { if (iFlag >= iTimes) { - this.server.abort("verifyAndAssignRoot failed after" + iTimes + this.server.abort("verifyAndAssignMeta failed after" + iTimes + " times retries, aborting", e); throw new IOException("Aborting", e); } @@ -148,13 +148,6 @@ public class ServerShutdownHandler extends EventHandler { } /** - * @return True if the server we are processing was carrying -ROOT- - */ - boolean isCarryingRoot() { - return false; - } - - /** * @return True if the server we are processing was carrying .META. */ boolean isCarryingMeta() { @@ -188,27 +181,18 @@ public class ServerShutdownHandler extends EventHandler { serverName + ", will retry", ioe); } - // Assign root and meta if we were carrying them. - if (isCarryingRoot()) { // -ROOT- - LOG.info("Server " + serverName + - " was carrying ROOT. Trying to assign."); - this.services.getAssignmentManager(). - regionOffline(HRegionInfo.ROOT_REGIONINFO); - verifyAndAssignRootWithRetries(); - } - - // Carrying meta? - if (isCarryingMeta()) { + // Assign meta if we were carrying it. + if (isCarryingMeta()) { // .META. LOG.info("Server " + serverName + - " was carrying META. Trying to assign."); + " was carrying META. Trying to assign."); this.services.getAssignmentManager(). regionOffline(HRegionInfo.FIRST_META_REGIONINFO); - this.services.getAssignmentManager().assignMeta(); + verifyAndAssignMetaWithRetries(); } // We don't want worker thread in the MetaServerShutdownHandler - // executor pool to block by waiting availability of -ROOT- - // and .META. server. Otherwise, it could run into the following issue: + // executor pool to block by waiting availability of .META. + // Otherwise, it could run into the following issue: // 1. The current MetaServerShutdownHandler instance For RS1 waits for the .META. // to come online. // 2. The newly assigned .META. region server RS2 was shutdown right after @@ -227,7 +211,7 @@ public class ServerShutdownHandler extends EventHandler { // If AssignmentManager hasn't finished rebuilding user regions, // we are not ready to assign dead regions either. So we re-queue up // the dead server for further processing too. - if (isCarryingRoot() || isCarryingMeta() // -ROOT- or .META. + if (isCarryingMeta() // .META. || !services.getAssignmentManager().isFailoverCleanupDone()) { this.services.getServerManager().processDeadServer(serverName); return; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 0651ef5..0e9a59a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -3956,13 +3956,14 @@ public class HRegion implements HeapSize { // , Writable{ /** * Inserts a new region's meta information into the passed * meta region. Used by the HMaster bootstrap code adding - * new table to ROOT table. + * new table to META table. * * @param meta META HRegion to be updated * @param r HRegion to add to meta * * @throws IOException */ + // TODO remove since only test and merge use this public static void addRegionToMETA(HRegion meta, HRegion r) throws IOException { meta.checkResources(); @@ -5148,13 +5149,9 @@ public class HRegion implements HeapSize { // , Writable{ final boolean majorCompact) throws IOException { HRegion region = null; - String rootStr = Bytes.toString(HConstants.ROOT_TABLE_NAME); String metaStr = Bytes.toString(HConstants.META_TABLE_NAME); // Currently expects tables have one region only. - if (p.getName().startsWith(rootStr)) { - region = HRegion.newHRegion(p, log, fs, c, HRegionInfo.ROOT_REGIONINFO, - HTableDescriptor.ROOT_TABLEDESC, null); - } else if (p.getName().startsWith(metaStr)) { + if (p.getName().startsWith(metaStr)) { region = HRegion.newHRegion(p, log, fs, c, HRegionInfo.FIRST_META_REGIONINFO, HTableDescriptor.META_TABLEDESC, null); } else { @@ -5221,10 +5218,10 @@ public class HRegion implements HeapSize { // , Writable{ * is based on the size of the store. */ public byte[] checkSplit() { - // Can't split ROOT/META + // Can't split META if (this.regionInfo.isMetaTable()) { if (shouldForceSplit()) { - LOG.warn("Cannot split root/meta regions in HBase 0.20 and above"); + LOG.warn("Cannot split meta region in HBase 0.20 and above"); } return null; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index bcde848..a191424 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -47,7 +47,6 @@ import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentSkipListMap; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantReadWriteLock; import javax.management.ObjectName; @@ -182,10 +181,8 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.regionserver.handler.CloseMetaHandler; import org.apache.hadoop.hbase.regionserver.handler.CloseRegionHandler; -import org.apache.hadoop.hbase.regionserver.handler.CloseRootHandler; import org.apache.hadoop.hbase.regionserver.handler.OpenMetaHandler; import org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler; -import org.apache.hadoop.hbase.regionserver.handler.OpenRootHandler; import org.apache.hadoop.hbase.regionserver.wal.HLog; import org.apache.hadoop.hbase.regionserver.wal.HLogUtil; import org.apache.hadoop.hbase.regionserver.wal.HLogFactory; @@ -204,11 +201,11 @@ import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.VersionInfo; import org.apache.hadoop.hbase.zookeeper.ClusterStatusTracker; import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; -import org.apache.hadoop.hbase.zookeeper.RootRegionTracker; import org.apache.hadoop.hbase.zookeeper.ZKClusterId; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperNodeTracker; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; +import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.metrics.util.MBeanUtil; import org.apache.hadoop.net.DNS; @@ -316,7 +313,6 @@ public class HRegionServer implements ClientProtocol, protected final Configuration conf; - protected final AtomicBoolean haveRootRegion = new AtomicBoolean(false); private boolean useHBaseChecksum; // verify hbase checksums? private Path rootDir; @@ -560,7 +556,7 @@ public class HRegionServer implements ClientProtocol, /** * Utility used ensuring higher quality of service for priority rpcs; e.g. - * rpcs to .META. and -ROOT-, etc. + * rpcs to .META., etc. */ class QosFunction implements Function { private final Map annotatedQos; @@ -939,7 +935,7 @@ public class HRegionServer implements ClientProtocol, LOG.info("stopping server " + this.serverNameFromMasterPOV); } // Interrupt catalog tracker here in case any regions being opened out in - // handlers are stuck waiting on meta or root. + // handlers are stuck waiting on meta. if (this.catalogTracker != null) this.catalogTracker.stop(); // Closing the compactSplit thread before closing meta regions @@ -991,8 +987,7 @@ public class HRegionServer implements ClientProtocol, } private boolean containsMetaTableRegions() { - return onlineRegions.containsKey(HRegionInfo.ROOT_REGIONINFO.getEncodedName()) - || onlineRegions.containsKey(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName()); + return onlineRegions.containsKey(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName()); } private boolean areAllUserRegionsOffline() { @@ -1464,14 +1459,10 @@ public class HRegionServer implements ClientProtocol, this.service = new ExecutorService(getServerName().toString()); this.service.startExecutorService(ExecutorType.RS_OPEN_REGION, conf.getInt("hbase.regionserver.executor.openregion.threads", 3)); - this.service.startExecutorService(ExecutorType.RS_OPEN_ROOT, - conf.getInt("hbase.regionserver.executor.openroot.threads", 1)); this.service.startExecutorService(ExecutorType.RS_OPEN_META, conf.getInt("hbase.regionserver.executor.openmeta.threads", 1)); this.service.startExecutorService(ExecutorType.RS_CLOSE_REGION, conf.getInt("hbase.regionserver.executor.closeregion.threads", 3)); - this.service.startExecutorService(ExecutorType.RS_CLOSE_ROOT, - conf.getInt("hbase.regionserver.executor.closeroot.threads", 1)); this.service.startExecutorService(ExecutorType.RS_CLOSE_META, conf.getInt("hbase.regionserver.executor.closemeta.threads", 1)); @@ -1599,13 +1590,10 @@ public class HRegionServer implements ClientProtocol, getCompactionRequester().requestCompaction(r, s, "Opening Region"); } } - // Update ZK, ROOT or META - if (r.getRegionInfo().isRootRegion()) { - RootRegionTracker.setRootLocation(getZooKeeper(), - this.serverNameFromMasterPOV); - } else if (r.getRegionInfo().isMetaRegion()) { - MetaEditor.updateMetaLocation(ct, r.getRegionInfo(), - this.serverNameFromMasterPOV); + // Update ZK, or META + if (r.getRegionInfo().isMetaRegion()) { + MetaRegionTracker.setMetaLocation(getZooKeeper(), + this.serverNameFromMasterPOV); } else { if (daughter) { // If daughter of a split, update whole row, not just location. @@ -1868,28 +1856,24 @@ public class HRegionServer implements ClientProtocol, } /** - * Close root and meta regions if we carry them + * Close meta region if we carry it * @param abort Whether we're running an abort. */ void closeMetaTableRegions(final boolean abort) { HRegion meta = null; - HRegion root = null; this.lock.writeLock().lock(); try { for (Map.Entry e: onlineRegions.entrySet()) { HRegionInfo hri = e.getValue().getRegionInfo(); - if (hri.isRootRegion()) { - root = e.getValue(); - } else if (hri.isMetaRegion()) { + if (hri.isMetaRegion()) { meta = e.getValue(); } - if (meta != null && root != null) break; + if (meta != null) break; } } finally { this.lock.writeLock().unlock(); } if (meta != null) closeRegion(meta.getRegionInfo(), abort, false); - if (root != null) closeRegion(root.getRegionInfo(), abort, false); } /** @@ -2405,10 +2389,7 @@ public class HRegionServer implements ClientProtocol, } this.regionsInTransitionInRS.putIfAbsent(region.getEncodedNameAsBytes(), false); CloseRegionHandler crh = null; - if (region.isRootRegion()) { - crh = new CloseRootHandler(this, this, region, abort, zk, - versionOfClosingNode); - } else if (region.isMetaRegion()) { + if (region.isMetaRegion()) { crh = new CloseMetaHandler(this, this, region, abort, zk, versionOfClosingNode); } else { @@ -3405,10 +3386,7 @@ public class HRegionServer implements ClientProtocol, this.regionsInTransitionInRS.putIfAbsent( region.getEncodedNameAsBytes(), true); // Need to pass the expected version in the constructor. - if (region.isRootRegion()) { - this.service.submit(new OpenRootHandler(this, this, region, htd, - versionOfOfflineNode)); - } else if (region.isMetaRegion()) { + if (region.isMetaRegion()) { this.service.submit(new OpenMetaHandler(this, this, region, htd, versionOfOfflineNode)); } else { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java index 074ef8f..2cfe205 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/OpenRegionHandler.java @@ -155,7 +155,7 @@ public class OpenRegionHandler extends EventHandler { } /** - * Update ZK, ROOT or META. This can take a while if for example the + * Update ZK or META. This can take a while if for example the * .META. is not available -- if server hosting .META. crashed and we are * waiting on it to come back -- so run in a thread and keep updating znode * state meantime so master doesn't timeout our region-in-transition. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java index f08738b..f1c0c44 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java @@ -138,6 +138,7 @@ public class ReplicationSource extends Thread private volatile boolean running = true; // Metrics for this source private MetricsSource metrics; + private int numLogsForWarn; /** * Instantiation method used by region servers @@ -185,6 +186,7 @@ public class ReplicationSource extends Thread this.conf.getLong("replication.source.sleepforretries", 1000); this.fs = fs; this.metrics = new MetricsSource(peerClusterZnode); + this.numLogsForWarn = this.conf.getInt("replication.source.numlogsforwarn", 2); try { this.clusterId = zkHelper.getUUIDForCluster(zkHelper.getZookeeperWatcher()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java index 2dd9771..31626f9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java @@ -261,9 +261,9 @@ public class AccessController extends BaseRegionObserver HRegionInfo hri = e.getRegion().getRegionInfo(); byte[] tableName = hri.getTableName(); - // 1. All users need read access to .META. and -ROOT- tables. + // 1. All users need read access to .META. table. // this is a very common operation, so deal with it quickly. - if (hri.isRootRegion() || hri.isMetaRegion()) { + if (hri.isMetaRegion()) { if (permRequest == Permission.Action.READ) { return AuthResult.allow("All users allowed", user, permRequest, tableName); } @@ -279,7 +279,7 @@ public class AccessController extends BaseRegionObserver // e.g. When a table is removed an entry is removed from .META. and _acl_ // and the user need to be allowed to write on both tables. if (permRequest == Permission.Action.WRITE && - (hri.isRootRegion() || hri.isMetaRegion() || + (hri.isMetaRegion() || Bytes.equals(tableName, AccessControlLists.ACL_GLOBAL_NAME)) && (authManager.authorize(user, Permission.Action.CREATE) || authManager.authorize(user, Permission.Action.ADMIN))) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java index f6c2886..27f55b4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSUtils.java @@ -373,7 +373,7 @@ public abstract class FSUtils { throws IOException, DeserializationException { String version = getVersion(fs, rootdir); if (version == null) { - if (!rootRegionExists(fs, rootdir)) { + if (!metaRegionExists(fs, rootdir)) { // rootDir is empty (no version file and no root region) // just create new version file (HBASE-1195) setVersion(fs, rootdir, wait, retries); @@ -672,10 +672,10 @@ public abstract class FSUtils { * @return true if exists * @throws IOException e */ - public static boolean rootRegionExists(FileSystem fs, Path rootdir) + public static boolean metaRegionExists(FileSystem fs, Path rootdir) throws IOException { Path rootRegionDir = - HRegion.getRegionDir(rootdir, HRegionInfo.ROOT_REGIONINFO); + HRegion.getRegionDir(rootdir, HRegionInfo.FIRST_META_REGIONINFO); return fs.exists(rootRegionDir); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java index 2934bfd..b391293 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java @@ -91,7 +91,7 @@ import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE; import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker; import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandler; import org.apache.hadoop.hbase.util.hbck.TableIntegrityErrorHandlerImpl; -import org.apache.hadoop.hbase.zookeeper.RootRegionTracker; +import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker; import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.security.AccessControlException; @@ -193,7 +193,7 @@ public class HBaseFsck extends Configured implements Tool { private boolean fixSplitParents = false; // fix lingering split parents // limit checking/fixes to listed tables, if empty attempt to check/fix all - // -ROOT- and .META. are always checked + // .META. are always checked private Set tablesIncluded = new HashSet(); private int maxMerge = DEFAULT_MAX_MERGE; // maximum number of overlapping regions to merge private int maxOverlapsToSideline = DEFAULT_OVERLAPS_TO_SIDELINE; // maximum number of overlapping regions to sideline @@ -230,7 +230,7 @@ public class HBaseFsck extends Configured implements Tool { * If tablesIncluded is empty, this map contains all tables. * Otherwise, it contains only meta tables and tables in tablesIncluded, * unless checkMetaOnly is specified, in which case, it contains only - * the meta tables (.META. and -ROOT-). + * the meta table */ private SortedMap tablesInfo = new ConcurrentSkipListMap(); @@ -254,7 +254,7 @@ public class HBaseFsck extends Configured implements Tool { errors = getErrorReporter(conf); int numThreads = conf.getInt("hbasefsck.numthreads", MAX_NUM_THREADS); - executor = new ScheduledThreadPoolExecutor(numThreads); + executor = new ScheduledThreadPoolExecutor(numThreads, Threads.newDaemonThreadFactory("hbasefsck")); } /** @@ -625,8 +625,7 @@ public class HBaseFsck extends Configured implements Tool { String tableName = td.getNameAsString(); errors.detail(" Table: " + tableName + "\t" + (td.isReadOnly() ? "ro" : "rw") + "\t" + - (td.isRootRegion() ? "ROOT" : - (td.isMetaRegion() ? "META" : " ")) + "\t" + + (td.isMetaRegion() ? "META" : " ") + "\t" + " families: " + td.getFamilies().size()); } } @@ -847,23 +846,14 @@ public class HBaseFsck extends Configured implements Tool { * * @return an open .META. HRegion */ - private HRegion createNewRootAndMeta() throws IOException { + private HRegion createNewMeta() throws IOException { Path rootdir = new Path(getConf().get(HConstants.HBASE_DIR)); Configuration c = getConf(); - HRegionInfo rootHRI = new HRegionInfo(HRegionInfo.ROOT_REGIONINFO); - MasterFileSystem.setInfoFamilyCachingForRoot(false); HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO); MasterFileSystem.setInfoFamilyCachingForMeta(false); - HRegion root = HRegion.createHRegion(rootHRI, rootdir, c, - HTableDescriptor.ROOT_TABLEDESC); HRegion meta = HRegion.createHRegion(metaHRI, rootdir, c, HTableDescriptor.META_TABLEDESC); - MasterFileSystem.setInfoFamilyCachingForRoot(true); MasterFileSystem.setInfoFamilyCachingForMeta(true); - - // Add first region from the META table to the ROOT region. - HRegion.addRegionToMETA(root, meta); - HRegion.closeHRegion(root); return meta; } @@ -879,9 +869,8 @@ public class HBaseFsck extends Configured implements Tool { for (Entry e : tablesInfo.entrySet()) { String name = e.getKey(); - // skip "-ROOT-" and ".META." - if (Bytes.compareTo(Bytes.toBytes(name), HConstants.ROOT_TABLE_NAME) == 0 - || Bytes.compareTo(Bytes.toBytes(name), HConstants.META_TABLE_NAME) == 0) { + // skip ".META." + if (Bytes.compareTo(Bytes.toBytes(name), HConstants.META_TABLE_NAME) == 0) { continue; } @@ -961,24 +950,24 @@ public class HBaseFsck extends Configured implements Tool { } } - // we can rebuild, move old root and meta out of the way and start + // we can rebuild, move old meta out of the way and start LOG.info("HDFS regioninfo's seems good. Sidelining old .META."); - Path backupDir = sidelineOldRootAndMeta(); + Path backupDir = sidelineOldMeta(); LOG.info("Creating new .META."); - HRegion meta = createNewRootAndMeta(); + HRegion meta = createNewMeta(); // populate meta List puts = generatePuts(tablesInfo); if (puts == null) { LOG.fatal("Problem encountered when creating new .META. entries. " + - "You may need to restore the previously sidelined -ROOT- and .META."); + "You may need to restore the previously sidelined .META."); return false; } meta.put(puts.toArray(new Put[0])); HRegion.closeHRegion(meta); LOG.info("Success! .META. table rebuilt."); - LOG.info("Old -ROOT- and .META. are moved into " + backupDir); + LOG.info("Old .META. is moved into " + backupDir); return true; } @@ -1118,29 +1107,19 @@ public class HBaseFsck extends Configured implements Tool { /** * @return Path to backup of original directory */ - Path sidelineOldRootAndMeta() throws IOException { - // put current -ROOT- and .META. aside. + Path sidelineOldMeta() throws IOException { + // put current .META. aside. Path hbaseDir = new Path(getConf().get(HConstants.HBASE_DIR)); FileSystem fs = hbaseDir.getFileSystem(getConf()); Path backupDir = getSidelineDir(); fs.mkdirs(backupDir); - sidelineTable(fs, HConstants.ROOT_TABLE_NAME, hbaseDir, backupDir); try { sidelineTable(fs, HConstants.META_TABLE_NAME, hbaseDir, backupDir); } catch (IOException e) { - LOG.error("Attempt to sideline meta failed, attempt to revert...", e); - try { - // move it back. - sidelineTable(fs, HConstants.ROOT_TABLE_NAME, backupDir, hbaseDir); - LOG.warn("... revert succeed. -ROOT- and .META. still in " - + "original state."); - } catch (IOException ioe) { - LOG.fatal("... failed to sideline root and meta and failed to restore " - + "prevoius state. Currently in inconsistent state. To restore " - + "try to rename -ROOT- in " + backupDir.getName() + " to " - + hbaseDir.getName() + ".", ioe); - } + LOG.fatal("... failed to sideline meta. Currently in inconsistent state. To restore " + + "try to rename .META. in " + backupDir.getName() + " to " + + hbaseDir.getName() + ".", e); throw e; // throw original exception } return backupDir; @@ -1197,7 +1176,6 @@ public class HBaseFsck extends Configured implements Tool { foundVersionFile = true; } else { if ((!checkMetaOnly && isTableIncluded(dirName)) || - dirName.equals("-ROOT-") || dirName.equals(".META.")) { tableDirs.add(file); } @@ -1241,31 +1219,29 @@ public class HBaseFsck extends Configured implements Tool { } /** - * Record the location of the ROOT region as found in ZooKeeper, - * as if it were in a META table. This is so that we can check - * deployment of ROOT. + * Record the location of the META region as found in ZooKeeper. */ - private boolean recordRootRegion() throws IOException { - HRegionLocation rootLocation = connection.locateRegion( - HConstants.ROOT_TABLE_NAME, HConstants.EMPTY_START_ROW); + private boolean recordMetaRegion() throws IOException { + HRegionLocation metaLocation = connection.locateRegion( + HConstants.META_TABLE_NAME, HConstants.EMPTY_START_ROW); - // Check if Root region is valid and existing - if (rootLocation == null || rootLocation.getRegionInfo() == null || - rootLocation.getHostname() == null) { - errors.reportError(ERROR_CODE.NULL_ROOT_REGION, - "Root Region or some of its attributes are null."); + // Check if Meta region is valid and existing + if (metaLocation == null || metaLocation.getRegionInfo() == null || + metaLocation.getHostname() == null) { + errors.reportError(ERROR_CODE.NULL_META_REGION, + "META region or some of its attributes are null."); return false; } ServerName sn; try { - sn = getRootRegionServerName(); + sn = getMetaRegionServerName(); } catch (KeeperException e) { throw new IOException(e); } MetaEntry m = - new MetaEntry(rootLocation.getRegionInfo(), sn, System.currentTimeMillis()); + new MetaEntry(metaLocation.getRegionInfo(), sn, System.currentTimeMillis()); HbckInfo hbInfo = new HbckInfo(m); - regionInfoMap.put(rootLocation.getRegionInfo().getEncodedName(), hbInfo); + regionInfoMap.put(metaLocation.getRegionInfo().getEncodedName(), hbInfo); return true; } @@ -1285,12 +1261,12 @@ public class HBaseFsck extends Configured implements Tool { }); } - private ServerName getRootRegionServerName() + private ServerName getMetaRegionServerName() throws IOException, KeeperException { ZooKeeperWatcher zkw = createZooKeeperWatcher(); ServerName sn = null; try { - sn = RootRegionTracker.getRootRegionLocation(zkw); + sn = MetaRegionTracker.getMetaRegionLocation(zkw); } finally { zkw.close(); } @@ -2436,14 +2412,14 @@ public class HBaseFsck extends Configured implements Tool { } /** - * Scan .META. and -ROOT-, adding all regions found to the regionInfo map. + * Scan .META., adding all regions found to the regionInfo map. * @throws IOException if an error is encountered */ boolean loadMetaEntries() throws IOException { // get a list of all regions from the master. This involves // scanning the META table - if (!recordRootRegion()) { + if (!recordMetaRegion()) { // Will remove later if we can fix it errors.reportError("Fatal error: unable to get root region location. Exiting..."); return false; @@ -2475,7 +2451,7 @@ public class HBaseFsck extends Configured implements Tool { } HRegionInfo hri = pair.getFirst(); if (!(isTableIncluded(hri.getTableNameAsString()) - || hri.isMetaRegion() || hri.isRootRegion())) { + || hri.isMetaRegion())) { return true; } MetaEntry m = new MetaEntry(hri, sn, ts); @@ -2497,11 +2473,6 @@ public class HBaseFsck extends Configured implements Tool { } } }; - - // Scan -ROOT- to pick up META regions - MetaScanner.metaScan(getConf(), visitor, null, null, - Integer.MAX_VALUE, HConstants.ROOT_TABLE_NAME); - if (!checkMetaOnly) { // Scan .META. to pick up user regions MetaScanner.metaScan(getConf(), visitor); @@ -2766,7 +2737,7 @@ public class HBaseFsck extends Configured implements Tool { public interface ErrorReporter { public static enum ERROR_CODE { - UNKNOWN, NO_META_REGION, NULL_ROOT_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META, + UNKNOWN, NO_META_REGION, NULL_META_REGION, NO_VERSION_FILE, NOT_IN_META_HDFS, NOT_IN_META, NOT_IN_META_OR_DEPLOYED, NOT_IN_HDFS_OR_DEPLOYED, NOT_IN_HDFS, SERVER_DOES_NOT_MATCH_META, NOT_DEPLOYED, MULTI_DEPLOYED, SHOULD_NOT_BE_DEPLOYED, MULTI_META_REGION, RS_CONNECT_FAILURE, FIRST_REGION_STARTKEY_NOT_EMPTY, LAST_REGION_ENDKEY_NOT_EMPTY, DUPE_STARTKEYS, @@ -3295,8 +3266,8 @@ public class HBaseFsck extends Configured implements Tool { System.err.println(" -sleepBeforeRerun Sleep this many seconds" + " before checking if the fix worked if run with -fix"); System.err.println(" -summary Print only summary of the tables and status."); - System.err.println(" -metaonly Only check the state of ROOT and META tables."); - System.err.println(" -sidelineDir HDFS path to backup existing meta and root."); + System.err.println(" -metaonly Only check the state of META table."); + System.err.println(" -sidelineDir HDFS path to backup existing meta."); System.err.println(""); System.err.println(" Metadata Repair options: (expert features, use with caution!)"); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java index 777f5e2..d0d68d7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HMerge.java @@ -121,7 +121,7 @@ class HMerge { throw new IllegalStateException( "Can not compact META table if instance is on-line"); } - new OfflineMerger(conf, fs).process(); + // TODO reenable new OfflineMerger(conf, fs).process(); } else { if(!masterIsRunning) { throw new IllegalStateException( @@ -341,7 +341,7 @@ class HMerge { } /** Instantiated to compact the meta region */ - private static class OfflineMerger extends Merger { + /*private static class OfflineMerger extends Merger { private final List metaRegions = new ArrayList(); private final HRegion root; @@ -430,5 +430,5 @@ class HMerge { LOG.debug("updated columns in row: " + Bytes.toStringBinary(newRegion.getRegionName())); } } - } + }*/ } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/Merge.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/Merge.java index 85d365f..514505b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/Merge.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/Merge.java @@ -148,22 +148,22 @@ public class Merge extends Configured implements Tool { * regions will be needed, but it is included for completeness. */ private void mergeTwoMetaRegions() throws IOException { - HRegion rootRegion = utils.getRootRegion(); + HRegion metaRegion = utils.getMetaRegion(); Get get = new Get(region1); get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); - Result result1 = rootRegion.get(get, null); + Result result1 = metaRegion.get(get, null); Preconditions.checkState(!result1.isEmpty(), "First region cells can not be null"); HRegionInfo info1 = HRegionInfo.getHRegionInfo(result1); get = new Get(region2); get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); - Result result2 = rootRegion.get(get, null); + Result result2 = metaRegion.get(get, null); Preconditions.checkState(!result2.isEmpty(), "Second region cells can not be null"); HRegionInfo info2 = HRegionInfo.getHRegionInfo(result2); - HRegion merged = merge(HTableDescriptor.META_TABLEDESC, info1, rootRegion, info2, rootRegion); + HRegion merged = merge(HTableDescriptor.META_TABLEDESC, info1, metaRegion, info2, metaRegion); LOG.info("Adding " + merged.getRegionInfo() + " to " + - rootRegion.getRegionInfo()); - HRegion.addRegionToMETA(rootRegion, merged); + metaRegion.getRegionInfo()); + HRegion.addRegionToMETA(metaRegion, merged); merged.close(); } @@ -219,7 +219,7 @@ public class Merge extends Configured implements Tool { } LOG.info("Found meta for region1 " + Bytes.toStringBinary(meta1.getRegionName()) + ", meta for region2 " + Bytes.toStringBinary(meta2.getRegionName())); - HRegion metaRegion1 = this.utils.getMetaRegion(meta1); + HRegion metaRegion1 = this.utils.getMetaRegion(); Get get = new Get(region1); get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); Result result1 = metaRegion1.get(get, null); @@ -235,7 +235,7 @@ public class Merge extends Configured implements Tool { if (Bytes.equals(meta1.getRegionName(), meta2.getRegionName())) { metaRegion2 = metaRegion1; } else { - metaRegion2 = utils.getMetaRegion(meta2); + metaRegion2 = utils.getMetaRegion(); } get = new Get(region2); get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER); @@ -265,7 +265,7 @@ public class Merge extends Configured implements Tool { } else if (Bytes.equals(mergedInfo.getRegionName(), meta2.getRegionName())) { mergeMeta = metaRegion2; } else { - mergeMeta = utils.getMetaRegion(mergedInfo); + mergeMeta = utils.getMetaRegion(); } LOG.info("Adding " + merged.getRegionInfo() + " to " + mergeMeta.getRegionInfo()); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java index b6fb9b0..6886521 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/MetaUtils.java @@ -62,7 +62,7 @@ public class MetaUtils { private final Configuration conf; private FileSystem fs; private HLog log; - private HRegion rootRegion; + private HRegion metaRegion; private Map metaRegions = Collections.synchronizedSortedMap( new TreeMap(Bytes.BYTES_COMPARATOR)); @@ -80,7 +80,7 @@ public class MetaUtils { public MetaUtils(Configuration conf) throws IOException { this.conf = conf; conf.setInt("hbase.client.retries.number", 1); - this.rootRegion = null; + this.metaRegion = null; initialize(); } @@ -107,31 +107,14 @@ public class MetaUtils { } /** - * @return HRegion for root region + * @return HRegion for meta region * @throws IOException e */ - public HRegion getRootRegion() throws IOException { - if (this.rootRegion == null) { - openRootRegion(); + public HRegion getMetaRegion() throws IOException { + if (this.metaRegion == null) { + openMetaRegion(); } - return this.rootRegion; - } - - /** - * Open or return cached opened meta region - * - * @param metaInfo HRegionInfo for meta region - * @return meta HRegion - * @throws IOException e - */ - public HRegion getMetaRegion(HRegionInfo metaInfo) throws IOException { - HRegion meta = metaRegions.get(metaInfo.getRegionName()); - if (meta == null) { - meta = openMetaRegion(metaInfo); - LOG.info("OPENING META " + meta.toString()); - this.metaRegions.put(metaInfo.getRegionName(), meta); - } - return meta; + return this.metaRegion; } /** @@ -140,13 +123,13 @@ public class MetaUtils { * MetaUtils edit session. */ public void shutdown() { - if (this.rootRegion != null) { + if (this.metaRegion != null) { try { - this.rootRegion.close(); + this.metaRegion.close(); } catch (IOException e) { - LOG.error("closing root region", e); + LOG.error("closing meta region", e); } finally { - this.rootRegion = null; + this.metaRegion = null; } } try { @@ -193,12 +176,13 @@ public class MetaUtils { * @param listener method to be called for each meta region found * @throws IOException e */ + // TODO the wiring here is all wrong public void scanRootRegion(ScannerListener listener) throws IOException { // Open root region so we can scan it - if (this.rootRegion == null) { - openRootRegion(); + if (this.metaRegion == null) { + openMetaRegion(); } - scanMetaRegion(this.rootRegion, listener); + scanMetaRegion(this.metaRegion, listener); } /** @@ -256,27 +240,27 @@ public class MetaUtils { ScannerListener listener) throws IOException { // Open meta region so we can scan it - HRegion metaRegion = openMetaRegion(metaRegionInfo); + HRegion metaRegion = openMetaRegion(); scanMetaRegion(metaRegion, listener); } - private synchronized HRegion openRootRegion() throws IOException { - if (this.rootRegion != null) { - return this.rootRegion; + private synchronized HRegion openMetaRegion() throws IOException { + if (this.metaRegion != null) { + return this.metaRegion; } - this.rootRegion = HRegion.openHRegion(HRegionInfo.ROOT_REGIONINFO, - HTableDescriptor.ROOT_TABLEDESC, getLog(), + this.metaRegion = HRegion.openHRegion(HRegionInfo.FIRST_META_REGIONINFO, + HTableDescriptor.META_TABLEDESC, getLog(), this.conf); - this.rootRegion.compactStores(); - return this.rootRegion; + this.metaRegion.compactStores(); + return this.metaRegion; } - private HRegion openMetaRegion(HRegionInfo metaInfo) throws IOException { + /*private HRegion openMetaRegion(HRegionInfo metaInfo) throws IOException { HRegion meta = HRegion.openHRegion(metaInfo, HTableDescriptor.META_TABLEDESC, getLog(), this.conf); meta.compactStores(); return meta; - } + }*/ /** * Set a single region on/offline. @@ -378,7 +362,7 @@ public class MetaUtils { final List result = new ArrayList(); // If passed table name is META, then return the root region. if (Bytes.equals(HConstants.META_TABLE_NAME, tableName)) { - result.add(openRootRegion().getRegionInfo()); + result.add(openMetaRegion().getRegionInfo()); return result; } // Return all meta regions that contain the passed tablename. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaRegionTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaRegionTracker.java new file mode 100644 index 0000000..6d3873b --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaRegionTracker.java @@ -0,0 +1,185 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.zookeeper; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.DeserializationException; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; +import org.apache.zookeeper.KeeperException; + +/** + * Tracks the meta region server location node in zookeeper. + * Meta region location is set by {@link MetaLocationEditor???} usually called + * out of RegionServerServices. + * This class has a watcher on the meta location and notices changes. + */ +@InterfaceAudience.Private +public class MetaRegionTracker extends ZooKeeperNodeTracker { + /** + * Creates a meta region location tracker. + * + *

    After construction, use {@link #start} to kick off tracking. + * + * @param watcher + * @param abortable + */ + public MetaRegionTracker(ZooKeeperWatcher watcher, Abortable abortable) { + super(watcher, watcher.metaServerZNode, abortable); + } + + /** + * Checks if the meta region location is available. + * @return true if meta region location is available, false if not + */ + public boolean isLocationAvailable() { + return super.getData(true) != null; + } + + /** + * Gets the meta region location, if available. Does not block. Sets a watcher. + * @return server name or null if we failed to get the data. + * @throws InterruptedException + */ + public ServerName getMetaRegionLocation() throws InterruptedException { + try { + return ServerName.parseFrom(super.getData(true)); + } catch (DeserializationException e) { + LOG.warn("Failed parse", e); + return null; + } + } + + /** + * Gets the meta region location, if available. Does not block. Does not set + * a watcher (In this regard it differs from {@link #getMetaRegionLocation}. + * @param zkw + * @return server name or null if we failed to get the data. + * @throws KeeperException + */ + public static ServerName getMetaRegionLocation(final ZooKeeperWatcher zkw) + throws KeeperException { + try { + return ServerName.parseFrom(ZKUtil.getData(zkw, zkw.metaServerZNode)); + } catch (DeserializationException e) { + throw ZKUtil.convert(e); + } + } + + /** + * Gets the meta region location, if available, and waits for up to the + * specified timeout if not immediately available. + * Given the zookeeper notification could be delayed, we will try to + * get the latest data. + * @param timeout maximum time to wait, in millis + * @return server name for server hosting meta region formatted as per + * {@link ServerName}, or null if none available + * @throws InterruptedException if interrupted while waiting + */ + public ServerName waitMetaRegionLocation(long timeout) + throws InterruptedException { + if (false == checkIfBaseNodeAvailable()) { + String errorMsg = "Check the value configured in 'zookeeper.znode.parent'. " + + "There could be a mismatch with the one configured in the master."; + LOG.error(errorMsg); + throw new IllegalArgumentException(errorMsg); + } + try { + return ServerName.parseFrom(super.blockUntilAvailable(timeout, true)); + } catch (DeserializationException e) { + LOG.warn("Failed parse", e); + return null; + } + } + + /** + * Sets the location of .META. in ZooKeeper to the + * specified server address. + * @param zookeeper zookeeper reference + * @param location The server hosting .META. + * @throws KeeperException unexpected zookeeper exception + */ + public static void setMetaLocation(ZooKeeperWatcher zookeeper, + final ServerName location) + throws KeeperException { + LOG.info("Setting META region location in ZooKeeper as " + location); + // Make the MetaRegionServer pb and then get its bytes and save this as + // the znode content. + byte [] data = toByteArray(location); + try { + ZKUtil.createAndWatch(zookeeper, zookeeper.metaServerZNode, data); + } catch(KeeperException.NodeExistsException nee) { + LOG.debug("META region location already existed, updated location"); + ZKUtil.setData(zookeeper, zookeeper.metaServerZNode, data); + } + } + + /** + * Build up the znode content. + * @param sn What to put into the znode. + * @return The content of the meta-region-server znode + */ + static byte [] toByteArray(final ServerName sn) { + // ZNode content is a pb message preceeded by some pb magic. + HBaseProtos.ServerName pbsn = + HBaseProtos.ServerName.newBuilder().setHostName(sn.getHostname()). + setPort(sn.getPort()).setStartCode(sn.getStartcode()).build(); + ZooKeeperProtos.RootRegionServer pbrsr = + ZooKeeperProtos.RootRegionServer.newBuilder().setServer(pbsn).build(); + return ProtobufUtil.prependPBMagic(pbrsr.toByteArray()); + } + + /** + * Deletes the location of .META. in ZooKeeper. + * @param zookeeper zookeeper reference + * @throws KeeperException unexpected zookeeper exception + */ + public static void deleteMetaLocation(ZooKeeperWatcher zookeeper) + throws KeeperException { + LOG.info("Unsetting META region location in ZooKeeper"); + try { + // Just delete the node. Don't need any watches. + ZKUtil.deleteNode(zookeeper, zookeeper.metaServerZNode); + } catch(KeeperException.NoNodeException nne) { + // Has already been deleted + } + } + + /** + * Wait until the meta region is available. + * @param zkw + * @param timeout + * @return ServerName or null if we timed out. + * @throws InterruptedException + */ + public static ServerName blockUntilAvailable(final ZooKeeperWatcher zkw, + final long timeout) + throws InterruptedException { + byte [] data = ZKUtil.blockUntilAvailable(zkw, zkw.metaServerZNode, timeout); + if (data == null) return null; + try { + return ServerName.parseFrom(data); + } catch (DeserializationException e) { + LOG.warn("Failed parse", e); + return null; + } + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java deleted file mode 100644 index 0808912..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/RootRegionTracker.java +++ /dev/null @@ -1,185 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.zookeeper; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hbase.Abortable; -import org.apache.hadoop.hbase.DeserializationException; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; -import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; -import org.apache.zookeeper.KeeperException; - -/** - * Tracks the root region server location node in zookeeper. - * Root region location is set by {@link RootLocationEditor} usually called - * out of RegionServerServices. - * This class has a watcher on the root location and notices changes. - */ -@InterfaceAudience.Private -public class RootRegionTracker extends ZooKeeperNodeTracker { - /** - * Creates a root region location tracker. - * - *

    After construction, use {@link #start} to kick off tracking. - * - * @param watcher - * @param abortable - */ - public RootRegionTracker(ZooKeeperWatcher watcher, Abortable abortable) { - super(watcher, watcher.rootServerZNode, abortable); - } - - /** - * Checks if the root region location is available. - * @return true if root region location is available, false if not - */ - public boolean isLocationAvailable() { - return super.getData(true) != null; - } - - /** - * Gets the root region location, if available. Does not block. Sets a watcher. - * @return server name or null if we failed to get the data. - * @throws InterruptedException - */ - public ServerName getRootRegionLocation() throws InterruptedException { - try { - return ServerName.parseFrom(super.getData(true)); - } catch (DeserializationException e) { - LOG.warn("Failed parse", e); - return null; - } - } - - /** - * Gets the root region location, if available. Does not block. Does not set - * a watcher (In this regard it differs from {@link #getRootRegionLocation()}. - * @param zkw - * @return server name or null if we failed to get the data. - * @throws KeeperException - */ - public static ServerName getRootRegionLocation(final ZooKeeperWatcher zkw) - throws KeeperException { - try { - return ServerName.parseFrom(ZKUtil.getData(zkw, zkw.rootServerZNode)); - } catch (DeserializationException e) { - throw ZKUtil.convert(e); - } - } - - /** - * Gets the root region location, if available, and waits for up to the - * specified timeout if not immediately available. - * Given the zookeeper notification could be delayed, we will try to - * get the latest data. - * @param timeout maximum time to wait, in millis - * @return server name for server hosting root region formatted as per - * {@link ServerName}, or null if none available - * @throws InterruptedException if interrupted while waiting - */ - public ServerName waitRootRegionLocation(long timeout) - throws InterruptedException { - if (false == checkIfBaseNodeAvailable()) { - String errorMsg = "Check the value configured in 'zookeeper.znode.parent'. " - + "There could be a mismatch with the one configured in the master."; - LOG.error(errorMsg); - throw new IllegalArgumentException(errorMsg); - } - try { - return ServerName.parseFrom(super.blockUntilAvailable(timeout, true)); - } catch (DeserializationException e) { - LOG.warn("Failed parse", e); - return null; - } - } - - /** - * Sets the location of -ROOT- in ZooKeeper to the - * specified server address. - * @param zookeeper zookeeper reference - * @param location The server hosting -ROOT- - * @throws KeeperException unexpected zookeeper exception - */ - public static void setRootLocation(ZooKeeperWatcher zookeeper, - final ServerName location) - throws KeeperException { - LOG.info("Setting ROOT region location in ZooKeeper as " + location); - // Make the RootRegionServer pb and then get its bytes and save this as - // the znode content. - byte [] data = toByteArray(location); - try { - ZKUtil.createAndWatch(zookeeper, zookeeper.rootServerZNode, data); - } catch(KeeperException.NodeExistsException nee) { - LOG.debug("ROOT region location already existed, updated location"); - ZKUtil.setData(zookeeper, zookeeper.rootServerZNode, data); - } - } - - /** - * Build up the znode content. - * @param sn What to put into the znode. - * @return The content of the root-region-server znode - */ - static byte [] toByteArray(final ServerName sn) { - // ZNode content is a pb message preceeded by some pb magic. - HBaseProtos.ServerName pbsn = - HBaseProtos.ServerName.newBuilder().setHostName(sn.getHostname()). - setPort(sn.getPort()).setStartCode(sn.getStartcode()).build(); - ZooKeeperProtos.RootRegionServer pbrsr = - ZooKeeperProtos.RootRegionServer.newBuilder().setServer(pbsn).build(); - return ProtobufUtil.prependPBMagic(pbrsr.toByteArray()); - } - - /** - * Deletes the location of -ROOT- in ZooKeeper. - * @param zookeeper zookeeper reference - * @throws KeeperException unexpected zookeeper exception - */ - public static void deleteRootLocation(ZooKeeperWatcher zookeeper) - throws KeeperException { - LOG.info("Unsetting ROOT region location in ZooKeeper"); - try { - // Just delete the node. Don't need any watches. - ZKUtil.deleteNode(zookeeper, zookeeper.rootServerZNode); - } catch(KeeperException.NoNodeException nne) { - // Has already been deleted - } - } - - /** - * Wait until the root region is available. - * @param zkw - * @param timeout - * @return ServerName or null if we timed out. - * @throws InterruptedException - */ - public static ServerName blockUntilAvailable(final ZooKeeperWatcher zkw, - final long timeout) - throws InterruptedException { - byte [] data = ZKUtil.blockUntilAvailable(zkw, zkw.rootServerZNode, timeout); - if (data == null) return null; - try { - return ServerName.parseFrom(data); - } catch (DeserializationException e) { - LOG.warn("Failed parse", e); - return null; - } - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java index 1a80b58..c540fa0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java @@ -725,7 +725,7 @@ public class ZKUtil { // Certain znodes are accessed directly by the client, // so they must be readable by non-authenticated clients if ((node.equals(zkw.baseZNode) == true) || - (node.equals(zkw.rootServerZNode) == true) || + (node.equals(zkw.metaServerZNode) == true) || (node.equals(zkw.getMasterAddressZNode()) == true) || (node.equals(zkw.clusterIdZNode) == true) || (node.equals(zkw.rsZNode) == true) || @@ -1062,7 +1062,7 @@ public class ZKUtil { zkw.backupMasterAddressesZNode)) { sb.append("\n ").append(child); } - sb.append("\nRegion server holding ROOT: " + RootRegionTracker.getRootRegionLocation(zkw)); + sb.append("\nRegion server holding .META.: " + MetaRegionTracker.getMetaRegionLocation(zkw)); sb.append("\nRegion servers:"); for (String child : listChildrenNoWatch(zkw, zkw.rsZNode)) { sb.append("\n ").append(child); @@ -1144,7 +1144,7 @@ public class ZKUtil { (data == null? "null": data.length == 0? "empty": ( znode.startsWith(zkw.assignmentZNode)? ZKAssign.toString(data): // We should not be doing this reaching into another class - znode.startsWith(zkw.rootServerZNode)? + znode.startsWith(zkw.metaServerZNode)? getServerNameOrEmptyString(data): znode.startsWith(zkw.backupMasterAddressesZNode)? getServerNameOrEmptyString(data): diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java index 128a0d9..6e66512 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java @@ -81,8 +81,8 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable { // base znode for this cluster public String baseZNode; - // znode containing location of server hosting root region - public String rootServerZNode; + // znode containing location of server hosting meta region + public String metaServerZNode; // znode containing ephemeral nodes of the regionservers public String rsZNode; // znode containing ephemeral nodes of the draining regionservers @@ -193,8 +193,8 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable { private void setNodeNames(Configuration conf) { baseZNode = conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); - rootServerZNode = ZKUtil.joinZNode(baseZNode, - conf.get("zookeeper.znode.rootserver", "root-region-server")); + metaServerZNode = ZKUtil.joinZNode(baseZNode, + conf.get("zookeeper.znode.metaserver", "meta-region-server")); rsZNode = ZKUtil.joinZNode(baseZNode, conf.get("zookeeper.znode.rs", "rs")); drainingZNode = ZKUtil.joinZNode(baseZNode, diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp index c24f5ed..dbf770f 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -41,7 +41,7 @@ String tableName = request.getParameter("name"); HTable table = new HTable(conf, tableName); String tableHeader = "

    Table Regions

    "; - ServerName rl = master.getCatalogTracker().getRootLocation(); + ServerName rl = master.getCatalogTracker().getMetaLocation(); boolean showFragmentation = conf.getBoolean("hbase.master.ui.fragmentation.enabled", false); boolean readOnly = conf.getBoolean("hbase.master.ui.readonly", false); Map frags = null; @@ -192,22 +192,7 @@ <% - if(tableName.equals(Bytes.toString(HConstants.ROOT_TABLE_NAME))) { -%> -<%= tableHeader %> -<% - String url = "http://" + rl.getHostname() + ":" + infoPort + "/"; -%> - - - - - - - -
    NameRegion ServerStart KeyEnd KeyRequests
    <%= tableName %><%= rl.getHostname() %>:<%= rl.getPort() %>--
    -<% - } else if(tableName.equals(Bytes.toString(HConstants.META_TABLE_NAME))) { + if(tableName.equals(Bytes.toString(HConstants.META_TABLE_NAME))) { %> <%= tableHeader %> <% diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java index 322e198..23be335 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java @@ -666,11 +666,8 @@ public abstract class HBaseTestCase extends TestCase { * @throws IOException */ protected void createRootAndMetaRegions() throws IOException { - root = HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, testDir, - conf, HTableDescriptor.ROOT_TABLEDESC); meta = HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, testDir, conf, HTableDescriptor.META_TABLEDESC); - HRegion.addRegionToMETA(root, meta); } protected void closeRootAndMeta() throws IOException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java index e889f41..a88f906 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java @@ -188,7 +188,7 @@ public class TestRegionRebalancing { if (!(avg > 2.0 && serverLoad <= avgLoadPlusSlop && serverLoad >= avgLoadMinusSlop)) { for (HRegionInfo hri : ProtobufUtil.getOnlineRegions(server)) { - if (hri.isMetaRegion() || hri.isRootRegion()) serverLoad--; + if (hri.isMetaRegion()) serverLoad--; // LOG.debug(hri.getRegionNameAsString()); } if (!(serverLoad <= avgLoadPlusSlop && serverLoad >= avgLoadMinusSlop)) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java index 046671c..441b78a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java @@ -53,7 +53,7 @@ import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetRequest; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.GetResponse; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; -import org.apache.hadoop.hbase.zookeeper.RootRegionTracker; +import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.util.Progressable; @@ -131,12 +131,12 @@ public class TestCatalogTracker { HConnection connection = Mockito.mock(HConnection.class); constructAndStartCatalogTracker(connection); try { - RootRegionTracker.setRootLocation(this.watcher, - new ServerName("example.com", 1234, System.currentTimeMillis())); + MetaRegionTracker.setMetaLocation(this.watcher, + new ServerName("example.com", 1234, System.currentTimeMillis())); } finally { // Clean out root location or later tests will be confused... they presume // start fresh in zk. - RootRegionTracker.deleteRootLocation(this.watcher); + MetaRegionTracker.deleteMetaLocation(this.watcher); } } @@ -154,7 +154,7 @@ public class TestCatalogTracker { Mockito.when(client.get((RpcController)Mockito.any(), (GetRequest)Mockito.any())). thenReturn(GetResponse.newBuilder().build()); final CatalogTracker ct = constructAndStartCatalogTracker(connection); - ServerName hsa = ct.getRootLocation(); + ServerName hsa = ct.getMetaLocation(); Assert.assertNull(hsa); ServerName meta = ct.getMetaLocation(); Assert.assertNull(meta); @@ -223,8 +223,8 @@ public class TestCatalogTracker { final CatalogTracker ct = constructAndStartCatalogTracker(connection); try { // Set a location for root and meta. - RootRegionTracker.setRootLocation(this.watcher, SN); - ct.setMetaLocation(SN); + MetaRegionTracker.setMetaLocation(this.watcher, SN); + // TODO wire this ct.setMetaLocation(SN); // Call the method that HBASE-4288 calls. It will try and verify the // meta location and will fail on first attempt then go into a long wait. // So, do this in a thread and then reset meta location to break it out @@ -246,14 +246,14 @@ public class TestCatalogTracker { while(!t.isAlive()) Threads.sleep(1); Threads.sleep(1); // Now reset the meta as though it were redeployed. - ct.setMetaLocation(SN); + // TODO wire this ct.setMetaLocation(SN); t.join(); Assert.assertTrue(metaSet.get()); } finally { // Clean out root and meta locations or later tests will be confused... // they presume start fresh in zk. - ct.resetMetaLocation(); - RootRegionTracker.deleteRootLocation(this.watcher); + // TODO wire this ct.resetMetaLocation(); + MetaRegionTracker.deleteMetaLocation(this.watcher); } } finally { // Clear out our doctored connection or could mess up subsequent tests. @@ -273,14 +273,14 @@ public class TestCatalogTracker { // Now start up the catalogtracker with our doctored Connection. final CatalogTracker ct = constructAndStartCatalogTracker(connection); try { - RootRegionTracker.setRootLocation(this.watcher, SN); + MetaRegionTracker.setMetaLocation(this.watcher, SN); long timeout = UTIL.getConfiguration(). getLong("hbase.catalog.verification.timeout", 1000); Assert.assertFalse(ct.verifyMetaRegionLocation(timeout)); } finally { // Clean out root location or later tests will be confused... they // presume start fresh in zk. - RootRegionTracker.deleteRootLocation(this.watcher); + MetaRegionTracker.deleteMetaLocation(this.watcher); } } finally { // Clear out our doctored connection or could mess up subsequent tests. @@ -343,13 +343,13 @@ public class TestCatalogTracker { thenReturn(implementation); final CatalogTracker ct = constructAndStartCatalogTracker(connection); try { - RootRegionTracker.setRootLocation(this.watcher, - new ServerName("example.com", 1234, System.currentTimeMillis())); - Assert.assertFalse(ct.verifyRootRegionLocation(100)); + MetaRegionTracker.setMetaLocation(this.watcher, + new ServerName("example.com", 1234, System.currentTimeMillis())); + Assert.assertFalse(ct.verifyMetaRegionLocation(100)); } finally { // Clean out root location or later tests will be confused... they presume // start fresh in zk. - RootRegionTracker.deleteRootLocation(this.watcher); + MetaRegionTracker.deleteMetaLocation(this.watcher); } } @@ -358,7 +358,7 @@ public class TestCatalogTracker { throws IOException, InterruptedException { HConnection connection = Mockito.mock(HConnection.class); final CatalogTracker ct = constructAndStartCatalogTracker(connection); - ct.waitForRoot(100); + ct.waitForMeta(100); } @Test (expected = RetriesExhaustedException.class) @@ -384,7 +384,7 @@ public class TestCatalogTracker { throws IOException, InterruptedException, KeeperException { HConnection connection = Mockito.mock(HConnection.class); final CatalogTracker ct = constructAndStartCatalogTracker(connection); - ServerName hsa = ct.getRootLocation(); + ServerName hsa = ct.getMetaLocation(); Assert.assertNull(hsa); // Now test waiting on root location getting set. @@ -395,11 +395,11 @@ public class TestCatalogTracker { // Join the thread... should exit shortly. t.join(); // Now root is available. - Assert.assertTrue(ct.getRootLocation().equals(hsa)); + Assert.assertTrue(ct.getMetaLocation().equals(hsa)); } private ServerName setRootLocation() throws KeeperException { - RootRegionTracker.setRootLocation(this.watcher, SN); + MetaRegionTracker.setMetaLocation(this.watcher, SN); return SN; } @@ -451,7 +451,7 @@ public class TestCatalogTracker { // This should trigger wake up of meta wait (Its the removal of the meta // region unassigned node that triggers catalogtrackers that a meta has // been assigned). - String node = ct.getMetaNodeTracker().getNode(); + String node = null; // TODO wire this ct.getMetaNodeTracker().getNode(); ZKUtil.createAndFailSilent(this.watcher, node); MetaEditor.updateMetaLocation(ct, HRegionInfo.FIRST_META_REGIONINFO, SN); ZKUtil.deleteNode(this.watcher, node); @@ -560,7 +560,7 @@ public class TestCatalogTracker { void doWaiting() throws InterruptedException { try { - while (this.ct.waitForRoot(100) == null); + while (this.ct.waitForMeta(100) == null); } catch (NotAllMetaRegionsOnlineException e) { // Ignore. } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTrackerOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTrackerOnCluster.java index 057085f..c9f72a0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTrackerOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTrackerOnCluster.java @@ -21,9 +21,8 @@ package org.apache.hadoop.hbase.catalog; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.zookeeper.RootRegionTracker; +import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; -import org.apache.zookeeper.KeeperException; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -61,7 +60,7 @@ public class TestCatalogTrackerOnCluster { }); ServerName nonsense = new ServerName("example.org", 1234, System.currentTimeMillis()); - RootRegionTracker.setRootLocation(zookeeper, nonsense); + MetaRegionTracker.setMetaLocation(zookeeper, nonsense); // Bring back up the hbase cluster. See if it can deal with nonsense root // location. The cluster should start and be fully available. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaMigrationConvertingToPB.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaMigrationConvertingToPB.java index 441a4d5..ae7eaf9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaMigrationConvertingToPB.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaMigrationConvertingToPB.java @@ -287,7 +287,7 @@ public class TestMetaMigrationConvertingToPB { p.add(HConstants.CATALOG_FAMILY, HConstants.META_VERSION_QUALIFIER, Bytes.toBytes(META_VERSION_092)); - MetaEditor.putToRootTable(ct, p); + // TODO wire this MetaEditor.putToRootTable(ct, p); LOG.info("Downgraded -ROOT- meta version=" + META_VERSION_092); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java index a458242..289fd0a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java @@ -43,7 +43,7 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.regionserver.RegionOpeningState; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Threads; -import org.apache.hadoop.hbase.zookeeper.RootRegionTracker; +import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.hbase.MediumTests; @@ -151,7 +151,7 @@ public class TestMasterNoCluster { // Put some data into the servers. Make it look like sn0 has the root // w/ an entry that points to sn1 as the host of .META. Put data into sn2 // so it looks like it has a few regions for a table named 't'. - RootRegionTracker.setRootLocation(rs0.getZooKeeper(), rs0.getServerName()); + MetaRegionTracker.setMetaLocation(rs0.getZooKeeper(), rs0.getServerName()); byte [] rootregion = Bytes.toBytes("-ROOT-,,0"); rs0.setGetResult(rootregion, HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), MetaMockingUtil.getMetaTableRowResult(HRegionInfo.FIRST_META_REGIONINFO, @@ -335,7 +335,7 @@ public class TestMasterNoCluster { // when its figured it just opened the root region by setting the root // location up into zk. Since we're mocking regionserver, need to do this // ourselves. - RootRegionTracker.setRootLocation(rs0.getZooKeeper(), rs0.getServerName()); + MetaRegionTracker.setMetaLocation(rs0.getZooKeeper(), rs0.getServerName()); // Do same transitions for .META. (presuming master has by now assigned // .META. to rs1). Mocking.fakeRegionServerRegionOpenInZK(master, rs0.getZooKeeper(), diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java index 9d1cc02..ab16019 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStatusServlet.java @@ -108,14 +108,14 @@ public class TestMasterStatusServlet { master, admin); } - @Test + /*@Test public void testStatusTemplateRootAvailable() throws IOException { new MasterStatusTmpl() .setRootLocation(new ServerName("rootserver:123,12345")) .render(new StringWriter(), master, admin); - } - + }*/ + /* @Test public void testStatusTemplateRootAndMetaAvailable() throws IOException { setupMockTables(); @@ -125,9 +125,9 @@ public class TestMasterStatusServlet { .setMetaLocation(new ServerName("metaserver:123,12345")) .render(new StringWriter(), master, admin); - } + }*/ - @Test + /*@Test public void testStatusTemplateWithServers() throws IOException { setupMockTables(); @@ -147,7 +147,7 @@ public class TestMasterStatusServlet { .setDeadServers(deadServers) .render(new StringWriter(), master, admin); - } + }*/ @Test public void testAssignmentManagerTruncatedList() throws IOException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java index 2550982..354d670 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java @@ -34,8 +34,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.*; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -102,6 +101,7 @@ public class TestHBaseFsck { private final static byte[] FAM = Bytes.toBytes(FAM_STR); private final static int REGION_ONLINE_TIMEOUT = 800; private static RegionStates regionStates; + private static ExecutorService executorService; // for the instance, reset every test run private HTable tbl; @@ -115,8 +115,13 @@ public class TestHBaseFsck { @BeforeClass public static void setUpBeforeClass() throws Exception { TEST_UTIL.getConfiguration().setBoolean(HConstants.DISTRIBUTED_LOG_SPLITTING_KEY, false); + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.handler.count", 2); + TEST_UTIL.getConfiguration().setInt("hbase.regionserver.metahandler.count", 2); TEST_UTIL.startMiniCluster(3); + executorService = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 60, TimeUnit.SECONDS, + new SynchronousQueue(), Threads.newDaemonThreadFactory("testhbck")); + AssignmentManager assignmentManager = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(); regionStates = assignmentManager.getRegionStates(); @@ -138,7 +143,7 @@ public class TestHBaseFsck { // Now let's mess it up and change the assignment in .META. to // point to a different region server - HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName()); + HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName(), executorService); ResultScanner scanner = meta.getScanner(new Scan()); HRegionInfo hri = null; @@ -178,7 +183,7 @@ public class TestHBaseFsck { assertNoErrors(doFsck(conf, false)); // comment needed - what is the purpose of this line - HTable t = new HTable(conf, Bytes.toBytes(table)); + HTable t = new HTable(conf, Bytes.toBytes(table), executorService); ResultScanner s = t.getScanner(new Scan()); s.close(); t.close(); @@ -193,7 +198,7 @@ public class TestHBaseFsck { private HRegionInfo createRegion(Configuration conf, final HTableDescriptor htd, byte[] startKey, byte[] endKey) throws IOException { - HTable meta = new HTable(conf, HConstants.META_TABLE_NAME); + HTable meta = new HTable(conf, HConstants.META_TABLE_NAME, executorService); HRegionInfo hri = new HRegionInfo(htd.getName(), startKey, endKey); MetaEditor.addRegionToMeta(meta, hri); meta.close(); @@ -285,7 +290,7 @@ public class TestHBaseFsck { } if (metaRow) { - HTable meta = new HTable(conf, HConstants.META_TABLE_NAME); + HTable meta = new HTable(conf, HConstants.META_TABLE_NAME, executorService); Delete delete = new Delete(deleteRow); meta.delete(delete); } @@ -310,7 +315,7 @@ public class TestHBaseFsck { HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM)); desc.addFamily(hcd); // If a table has no CF's it doesn't get checked TEST_UTIL.getHBaseAdmin().createTable(desc, SPLITS); - tbl = new HTable(TEST_UTIL.getConfiguration(), tablename); + tbl = new HTable(TEST_UTIL.getConfiguration(), Bytes.toBytes(tablename), executorService); List puts = new ArrayList(); for (byte[] row : ROWKEYS) { @@ -347,9 +352,16 @@ public class TestHBaseFsck { admin.getConnection().clearRegionCache(); byte[] tbytes = Bytes.toBytes(tablename); admin.disableTableAsync(tbytes); + long totalWait = 0; + long maxWait = 30*1000; + long sleepTime = 250; while (!admin.isTableDisabled(tbytes)) { try { - Thread.sleep(250); + Thread.sleep(sleepTime); + totalWait += sleepTime; + if (totalWait >= maxWait) { + fail("Waited too long for table to be disabled + " + tablename); + } } catch (InterruptedException e) { e.printStackTrace(); fail("Interrupted when trying to disable table " + tablename); @@ -719,7 +731,7 @@ public class TestHBaseFsck { assertNotNull(regionName); assertNotNull(serverName); - HTable meta = new HTable(conf, HConstants.META_TABLE_NAME); + HTable meta = new HTable(conf, HConstants.META_TABLE_NAME, executorService); Put put = new Put(regionName); put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes(serverName.getHostAndPort())); @@ -1207,7 +1219,7 @@ public class TestHBaseFsck { Bytes.toBytes("C"), true, true, false); // Create a new meta entry to fake it as a split parent. - HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName()); + HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getName(), executorService); HRegionInfo hri = location.getRegionInfo(); HRegionInfo a = new HRegionInfo(tbl.getTableName(), @@ -1267,7 +1279,7 @@ public class TestHBaseFsck { * This creates and fixes a bad table with a missing region which is the 1st region -- hole in * meta and data missing in the fs. */ - @Test + @Test(timeout=120000) public void testMissingFirstRegion() throws Exception { String table = "testMissingFirstRegion"; try { @@ -1295,7 +1307,7 @@ public class TestHBaseFsck { * This creates and fixes a bad table with missing last region -- hole in meta and data missing in * the fs. */ - @Test + @Test(timeout=120000) public void testMissingLastRegion() throws Exception { String table = "testMissingLastRegion"; try {