diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java index acd6ee7..b005221 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java @@ -25,6 +25,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; +import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.MetaRegionServer; import org.apache.hadoop.hbase.util.Addressing; import org.apache.hadoop.hbase.util.Bytes; @@ -159,6 +161,10 @@ public class ServerName implements Comparable, Serializable { return new ServerName(hostAndPort, startCode); } + public static ServerName valueOf(final org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName sn) { + return valueOf(sn.getHostName(), sn.getPort(), sn.getStartCode()); + } + @Override public String toString() { return getServerName(); @@ -355,9 +361,9 @@ public class ServerName implements Comparable, Serializable { if (ProtobufUtil.isPBMagicPrefix(data)) { int prefixLen = ProtobufUtil.lengthOfPBMagic(); try { - MetaRegionServer rss = - MetaRegionServer.PARSER.parseFrom(data, prefixLen, data.length - prefixLen); - org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName sn = rss.getServer(); + ZooKeeperProtos.Master rss = + ZooKeeperProtos.Master.PARSER.parseFrom(data, prefixLen, data.length - prefixLen); + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName sn = rss.getMaster(); return valueOf(sn.getHostName(), sn.getPort(), sn.getStartCode()); } catch (InvalidProtocolBufferException e) { // A failed parse of the znode is pretty catastrophic. Rather than loop @@ -382,4 +388,13 @@ public class ServerName implements Comparable, Serializable { int port = Addressing.parsePort(str); return valueOf(hostname, port, -1L); } + + public static HBaseProtos.ServerName toProtobuf(ServerName sn) { + return HBaseProtos.ServerName.newBuilder() + .setHostName(sn.getHostname()) + .setPort(sn.getPort()) + .setStartCode(sn.getStartcode()) + .build(); + + } } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java index d660db7..d010d72 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java @@ -52,7 +52,124 @@ public class RegionState { // region but hasn't be created yet, or master doesn't // know it's already created MERGING_NEW // new region to be created when RS merges two - // daughter regions but hasn't be created yet, or + ; + + /** + * Convert to protobuf ClusterStatusProtos.RegionState.State + */ + public ClusterStatusProtos.RegionState.State convert() { + ClusterStatusProtos.RegionState.State rs; + switch (this) { + case OFFLINE: + rs = ClusterStatusProtos.RegionState.State.OFFLINE; + break; + case PENDING_OPEN: + rs = ClusterStatusProtos.RegionState.State.PENDING_OPEN; + break; + case OPENING: + rs = ClusterStatusProtos.RegionState.State.OPENING; + break; + case OPEN: + rs = ClusterStatusProtos.RegionState.State.OPEN; + break; + case PENDING_CLOSE: + rs = ClusterStatusProtos.RegionState.State.PENDING_CLOSE; + break; + case CLOSING: + rs = ClusterStatusProtos.RegionState.State.CLOSING; + break; + case CLOSED: + rs = ClusterStatusProtos.RegionState.State.CLOSED; + break; + case SPLITTING: + rs = ClusterStatusProtos.RegionState.State.SPLITTING; + break; + case SPLIT: + rs = ClusterStatusProtos.RegionState.State.SPLIT; + break; + case FAILED_OPEN: + rs = ClusterStatusProtos.RegionState.State.FAILED_OPEN; + break; + case FAILED_CLOSE: + rs = ClusterStatusProtos.RegionState.State.FAILED_CLOSE; + break; + case MERGING: + rs = ClusterStatusProtos.RegionState.State.MERGING; + break; + case MERGED: + rs = ClusterStatusProtos.RegionState.State.MERGED; + break; + case SPLITTING_NEW: + rs = ClusterStatusProtos.RegionState.State.SPLITTING_NEW; + break; + case MERGING_NEW: + rs = ClusterStatusProtos.RegionState.State.MERGING_NEW; + break; + default: + throw new IllegalStateException(""); + } + return rs; + } + + /** + * Convert a protobuf HBaseProtos.RegionState.State to a RegionState.State + * + * @return the RegionState.State + */ + public static State convert(ClusterStatusProtos.RegionState.State protoState) { + State state; + switch (protoState) { + case OFFLINE: + state = OFFLINE; + break; + case PENDING_OPEN: + state = PENDING_OPEN; + break; + case OPENING: + state = OPENING; + break; + case OPEN: + state = OPEN; + break; + case PENDING_CLOSE: + state = PENDING_CLOSE; + break; + case CLOSING: + state = CLOSING; + break; + case CLOSED: + state = CLOSED; + break; + case SPLITTING: + state = SPLITTING; + break; + case SPLIT: + state = SPLIT; + break; + case FAILED_OPEN: + state = FAILED_OPEN; + break; + case FAILED_CLOSE: + state = FAILED_CLOSE; + break; + case MERGING: + state = MERGING; + break; + case MERGED: + state = MERGED; + break; + case SPLITTING_NEW: + state = SPLITTING_NEW; + break; + case MERGING_NEW: + state = MERGING_NEW; + break; + default: + throw new IllegalStateException(""); + } + return state; + } + // daughter regions but hasn't be created yet, or // master doesn't know it's already created } @@ -251,55 +368,8 @@ public class RegionState { public ClusterStatusProtos.RegionState convert() { ClusterStatusProtos.RegionState.Builder regionState = ClusterStatusProtos.RegionState.newBuilder(); ClusterStatusProtos.RegionState.State rs; - switch (this.state) { - case OFFLINE: - rs = ClusterStatusProtos.RegionState.State.OFFLINE; - break; - case PENDING_OPEN: - rs = ClusterStatusProtos.RegionState.State.PENDING_OPEN; - break; - case OPENING: - rs = ClusterStatusProtos.RegionState.State.OPENING; - break; - case OPEN: - rs = ClusterStatusProtos.RegionState.State.OPEN; - break; - case PENDING_CLOSE: - rs = ClusterStatusProtos.RegionState.State.PENDING_CLOSE; - break; - case CLOSING: - rs = ClusterStatusProtos.RegionState.State.CLOSING; - break; - case CLOSED: - rs = ClusterStatusProtos.RegionState.State.CLOSED; - break; - case SPLITTING: - rs = ClusterStatusProtos.RegionState.State.SPLITTING; - break; - case SPLIT: - rs = ClusterStatusProtos.RegionState.State.SPLIT; - break; - case FAILED_OPEN: - rs = ClusterStatusProtos.RegionState.State.FAILED_OPEN; - break; - case FAILED_CLOSE: - rs = ClusterStatusProtos.RegionState.State.FAILED_CLOSE; - break; - case MERGING: - rs = ClusterStatusProtos.RegionState.State.MERGING; - break; - case MERGED: - rs = ClusterStatusProtos.RegionState.State.MERGED; - break; - case SPLITTING_NEW: - rs = ClusterStatusProtos.RegionState.State.SPLITTING_NEW; - break; - case MERGING_NEW: - rs = ClusterStatusProtos.RegionState.State.MERGING_NEW; - break; - default: - throw new IllegalStateException(""); - } + State state = this.state; + rs = state.convert(); regionState.setRegionInfo(HRegionInfo.convert(hri)); regionState.setState(rs); regionState.setStamp(getStamp()); @@ -313,56 +383,8 @@ public class RegionState { */ public static RegionState convert(ClusterStatusProtos.RegionState proto) { RegionState.State state; - switch (proto.getState()) { - case OFFLINE: - state = State.OFFLINE; - break; - case PENDING_OPEN: - state = State.PENDING_OPEN; - break; - case OPENING: - state = State.OPENING; - break; - case OPEN: - state = State.OPEN; - break; - case PENDING_CLOSE: - state = State.PENDING_CLOSE; - break; - case CLOSING: - state = State.CLOSING; - break; - case CLOSED: - state = State.CLOSED; - break; - case SPLITTING: - state = State.SPLITTING; - break; - case SPLIT: - state = State.SPLIT; - break; - case FAILED_OPEN: - state = State.FAILED_OPEN; - break; - case FAILED_CLOSE: - state = State.FAILED_CLOSE; - break; - case MERGING: - state = State.MERGING; - break; - case MERGED: - state = State.MERGED; - break; - case SPLITTING_NEW: - state = State.SPLITTING_NEW; - break; - case MERGING_NEW: - state = State.MERGING_NEW; - break; - default: - throw new IllegalStateException(""); - } - + ClusterStatusProtos.RegionState.State protoState = proto.getState(); + state = State.convert(protoState); return new RegionState(HRegionInfo.convert(proto.getRegionInfo()),state,proto.getStamp(),null); } diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java index f1d17c2..269bced 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaTableLocator.java @@ -17,37 +17,40 @@ */ package org.apache.hadoop.hbase.zookeeper; +import javax.annotation.Nullable; +import java.io.EOFException; +import java.io.IOException; +import java.net.ConnectException; +import java.net.NoRouteToHostException; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.rmi.UnknownHostException; + +import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; +import com.google.protobuf.InvalidProtocolBufferException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.RetriesExhaustedException; import org.apache.hadoop.hbase.exceptions.DeserializationException; -import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ipc.RpcClient; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.ipc.RemoteException; import org.apache.zookeeper.KeeperException; -import java.io.EOFException; -import java.io.IOException; -import java.net.ConnectException; -import java.net.NoRouteToHostException; -import java.net.SocketException; -import java.net.SocketTimeoutException; -import java.rmi.UnknownHostException; - /** * Utility class to perform operation (get/wait for/verify/set/delete) on znode in ZooKeeper * which keeps hbase:meta region server location. @@ -74,18 +77,74 @@ public class MetaTableLocator { private volatile boolean stopped = false; /** + * Helper class for internal use. + * Contains meta region state and its location. + */ + @InterfaceAudience.Private + public static class MetaRegionLocation { + private final ServerName serverName; + private final RegionState.State state; + + /** + * Make instance of meta region location. + * @param serverName server, holding meta + * @param state state of the meta region + */ + public MetaRegionLocation(@Nullable ServerName serverName, RegionState.State state) { + Preconditions.checkArgument(serverName != null + || state.equals(RegionState.State.OFFLINE) + || state.equals(RegionState.State.CLOSED), + "Null server is allowed only in OFFLINE/CLOSED states"); + this.serverName = serverName; + this.state = state; + } + + /** + * Current holder of meta region. + * @return server name or null, if no holders + */ + @Nullable + public ServerName getServerName() { + return serverName; + } + + /** + * Current state of meta region. + * @return state + */ + public RegionState.State getState() { + return state; + } + + /** + * @return true if open + */ + public boolean isOnline() { + return state.equals(RegionState.State.OPEN); + } + + /** + * If meta was cleanly closed. + * @return true if closed or offline + */ + public boolean isOffline() { + return state.equals(RegionState.State.CLOSED) || state.equals(RegionState.State.OFFLINE); + } + } + + private static MetaRegionLocation NULL_LOCATION = new MetaRegionLocation(null, + RegionState.State.OFFLINE); + + /** * Checks if the meta region location is available. * @return true if meta region location is available, false if not */ public boolean isLocationAvailable(ZooKeeperWatcher zkw) { try { - return ZKUtil.getData(zkw, zkw.metaServerZNode) != null; + return getMetaRegionLocationAndState(zkw).isOnline(); } catch(KeeperException e) { LOG.error("ZK error trying to get hbase:meta from ZooKeeper"); return false; - } catch (InterruptedException e) { - LOG.error("ZK error trying to get hbase:meta from ZooKeeper"); - return false; } } @@ -94,18 +153,16 @@ public class MetaTableLocator { * @param zkw zookeeper connection to use * @return server name or null if we failed to get the data. */ + @Nullable public ServerName getMetaRegionLocation(final ZooKeeperWatcher zkw) { try { - try { - return ServerName.parseFrom(ZKUtil.getData(zkw, zkw.metaServerZNode)); - } catch (DeserializationException e) { - throw ZKUtil.convert(e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); + MetaRegionLocation location = getMetaRegionLocationAndState(zkw); + if (location.isOnline()) + return location.getServerName(); + else return null; - } } catch (KeeperException ke) { - return null; + return null; } } @@ -189,9 +246,8 @@ public class MetaTableLocator { } catch (RegionServerStoppedException e) { // Pass -- server name sends us to a server that is dying or already dead. } - return (service == null)? false: - verifyRegionLocation(service, - getMetaRegionLocation(zkw), META_REGION_NAME); + return (service != null) && verifyRegionLocation(service, + getMetaRegionLocation(zkw), META_REGION_NAME); } /** @@ -315,44 +371,38 @@ public class MetaTableLocator { * Sets the location of hbase:meta in ZooKeeper to the * specified server address. * @param zookeeper zookeeper reference - * @param location The server hosting hbase:meta + * @param serverName The server hosting hbase:meta * @throws KeeperException unexpected zookeeper exception */ - public static void setMetaLocation(ZooKeeperWatcher zookeeper, - final ServerName location) - throws KeeperException { - LOG.info("Setting hbase:meta region location in ZooKeeper as " + location); - // Make the MetaRegionServer pb and then get its bytes and save this as - // the znode content. - byte [] data = toByteArray(location); + public void setMetaLocation(ZooKeeperWatcher zookeeper, + ServerName serverName, RegionState.State state) throws IOException { try { - ZKUtil.createAndWatch(zookeeper, zookeeper.metaServerZNode, data); - } catch(KeeperException.NodeExistsException nee) { - LOG.debug("META region location already existed, updated location"); - ZKUtil.setData(zookeeper, zookeeper.metaServerZNode, data); + LOG.info("Setting hbase:meta region location in ZooKeeper as " + serverName); + // Make the MetaRegionServer pb and then get its bytes and save this as + // the znode content. + byte[] data = toZnodeBytes(serverName, state); + try { + ZKUtil.createAndWatch(zookeeper, zookeeper.metaServerZNode, data); + } catch (KeeperException.NodeExistsException nee) { + LOG.debug("META region location already existed, updated location"); + ZKUtil.setData(zookeeper, zookeeper.metaServerZNode, data); + } + } catch (KeeperException e) { + throw new IOException("Unable to set meta location", e); } } - /** - * Build up the znode content. - * @param sn What to put into the znode. - * @return The content of the meta-region-server znode - */ - private static byte [] toByteArray(final ServerName sn) { - // ZNode content is a pb message preceded by some pb magic. - HBaseProtos.ServerName pbsn = - HBaseProtos.ServerName.newBuilder() - .setHostName(sn.getHostname()) - .setPort(sn.getPort()) - .setStartCode(sn.getStartcode()) - .build(); - - ZooKeeperProtos.MetaRegionServer pbrsr = - ZooKeeperProtos.MetaRegionServer.newBuilder() - .setServer(pbsn) - .setRpcVersion(HConstants.RPC_CURRENT_VERSION) - .build(); - return ProtobufUtil.prependPBMagic(pbrsr.toByteArray()); + public MetaRegionLocation getMetaRegionLocationAndState(ZooKeeperWatcher zkw) + throws KeeperException { + try { + byte[] data = ZKUtil.getData(zkw, zkw.metaServerZNode); + return fromZnodeBytes(data); + } catch (DeserializationException e) { + throw ZKUtil.convert(e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return NULL_LOCATION; + } } /** @@ -362,7 +412,7 @@ public class MetaTableLocator { */ public void deleteMetaLocation(ZooKeeperWatcher zookeeper) throws KeeperException { - LOG.info("Unsetting hbase:meta region location in ZooKeeper"); + LOG.info("Deleting hbase:meta region location in ZooKeeper"); try { // Just delete the node. Don't need any watches. ZKUtil.deleteNode(zookeeper, zookeeper.metaServerZNode); @@ -372,7 +422,7 @@ public class MetaTableLocator { } /** - * Wait until the meta region is available. + * Wait until the meta region is available and is not in transition. * @param zkw zookeeper connection to use * @param timeout maximum time to wait, in millis * @return ServerName or null if we timed out. @@ -381,14 +431,28 @@ public class MetaTableLocator { public ServerName blockUntilAvailable(final ZooKeeperWatcher zkw, final long timeout) throws InterruptedException { - byte [] data = ZKUtil.blockUntilAvailable(zkw, zkw.metaServerZNode, timeout); - if (data == null) return null; - try { - return ServerName.parseFrom(data); - } catch (DeserializationException e) { - LOG.warn("Failed parse", e); - return null; - } + Stopwatch sw = new Stopwatch().start(); + ServerName sn = null; + do { + byte[] data = ZKUtil.blockUntilAvailable(zkw, zkw.metaServerZNode, timeout); + if (data == null) + break; + try { + MetaRegionLocation mrl = fromZnodeBytes(data); + if (mrl != null && mrl.getState().equals(RegionState.State.OPEN)) { + sn = mrl.getServerName(); + break; + } + if (sw.elapsedMillis() > timeout - HConstants.SOCKET_RETRY_WAIT_MS) + break; + Thread.sleep(HConstants.SOCKET_RETRY_WAIT_MS); + } catch (DeserializationException e) { + LOG.warn("Failed parse", e); + break; + } + } while (true); + sw.stop(); + return sn; } /** @@ -401,4 +465,45 @@ public class MetaTableLocator { stopped = true; } } + + + private MetaRegionLocation fromZnodeBytes(byte[] data) throws DeserializationException { + if (data == null || data.length == 0) + return NULL_LOCATION; + try { + if (ProtobufUtil.isPBMagicPrefix(data)) { + int prefixLen = ProtobufUtil.lengthOfPBMagic(); + ZooKeeperProtos.MetaRegionServer rl = + ZooKeeperProtos.MetaRegionServer.PARSER.parseFrom + (data, prefixLen, data.length - prefixLen); + return new MetaRegionLocation( + ServerName.valueOf(rl.getServer()), + RegionState.State.convert(rl.getState())); + } else { + // old style of meta region location? + // assume that if znode exists, it means that meta is online. + return new MetaRegionLocation(ServerName.parseFrom(data), RegionState.State.OPEN); + } + } catch (InvalidProtocolBufferException e) { + throw new DeserializationException("Unable to parse meat region location"); + } + } + + /** + * Build up the znode content. + * @param sn What to put into the znode. + * @return The content of the meta-region-server znode + */ + private static byte[] toZnodeBytes(ServerName sn, RegionState.State state) { + // ZNode content is a pb message preceded by some pb magic. + ZooKeeperProtos.MetaRegionServer pbrsr = + ZooKeeperProtos.MetaRegionServer.newBuilder() + .setServer(ServerName.toProtobuf(sn)) + .setRpcVersion(HConstants.RPC_CURRENT_VERSION) + .setState(state.convert()) + .build(); + return ProtobufUtil.prependPBMagic(pbrsr.toByteArray()); + } + + } diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java index 10274b4..6da497e 100644 --- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java @@ -16,7 +16,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * *
-     * The ServerName hosting the meta region currently.
+     * The ServerName hosting the meta region currently, or destination server,
+     * if meta region is in transition.
      * 
*/ boolean hasServer(); @@ -24,7 +25,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * *
-     * The ServerName hosting the meta region currently.
+     * The ServerName hosting the meta region currently, or destination server,
+     * if meta region is in transition.
      * 
*/ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName getServer(); @@ -32,7 +34,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * *
-     * The ServerName hosting the meta region currently.
+     * The ServerName hosting the meta region currently, or destination server,
+     * if meta region is in transition.
      * 
*/ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder getServerOrBuilder(); @@ -58,6 +61,24 @@ public final class ZooKeeperProtos { * */ int getRpcVersion(); + + // optional .RegionState.State state = 3; + /** + * optional .RegionState.State state = 3; + * + *
+     * State of the region transition. OPEN means fully operational 'hbase:meta'
+     * 
+ */ + boolean hasState(); + /** + * optional .RegionState.State state = 3; + * + *
+     * State of the region transition. OPEN means fully operational 'hbase:meta'
+     * 
+ */ + org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State getState(); } /** * Protobuf type {@code MetaRegionServer} @@ -133,6 +154,17 @@ public final class ZooKeeperProtos { rpcVersion_ = input.readUInt32(); break; } + case 24: { + int rawValue = input.readEnum(); + org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State value = org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State.valueOf(rawValue); + if (value == null) { + unknownFields.mergeVarintField(3, rawValue); + } else { + bitField0_ |= 0x00000004; + state_ = value; + } + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -180,7 +212,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * *
-     * The ServerName hosting the meta region currently.
+     * The ServerName hosting the meta region currently, or destination server,
+     * if meta region is in transition.
      * 
*/ public boolean hasServer() { @@ -190,7 +223,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * *
-     * The ServerName hosting the meta region currently.
+     * The ServerName hosting the meta region currently, or destination server,
+     * if meta region is in transition.
      * 
*/ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName getServer() { @@ -200,7 +234,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * *
-     * The ServerName hosting the meta region currently.
+     * The ServerName hosting the meta region currently, or destination server,
+     * if meta region is in transition.
      * 
*/ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder getServerOrBuilder() { @@ -235,9 +270,34 @@ public final class ZooKeeperProtos { return rpcVersion_; } + // optional .RegionState.State state = 3; + public static final int STATE_FIELD_NUMBER = 3; + private org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State state_; + /** + * optional .RegionState.State state = 3; + * + *
+     * State of the region transition. OPEN means fully operational 'hbase:meta'
+     * 
+ */ + public boolean hasState() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + /** + * optional .RegionState.State state = 3; + * + *
+     * State of the region transition. OPEN means fully operational 'hbase:meta'
+     * 
+ */ + public org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State getState() { + return state_; + } + private void initFields() { server_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.getDefaultInstance(); rpcVersion_ = 0; + state_ = org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State.OFFLINE; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -265,6 +325,9 @@ public final class ZooKeeperProtos { if (((bitField0_ & 0x00000002) == 0x00000002)) { output.writeUInt32(2, rpcVersion_); } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeEnum(3, state_.getNumber()); + } getUnknownFields().writeTo(output); } @@ -282,6 +345,10 @@ public final class ZooKeeperProtos { size += com.google.protobuf.CodedOutputStream .computeUInt32Size(2, rpcVersion_); } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeEnumSize(3, state_.getNumber()); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -315,6 +382,11 @@ public final class ZooKeeperProtos { result = result && (getRpcVersion() == other.getRpcVersion()); } + result = result && (hasState() == other.hasState()); + if (hasState()) { + result = result && + (getState() == other.getState()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -336,6 +408,10 @@ public final class ZooKeeperProtos { hash = (37 * hash) + RPC_VERSION_FIELD_NUMBER; hash = (53 * hash) + getRpcVersion(); } + if (hasState()) { + hash = (37 * hash) + STATE_FIELD_NUMBER; + hash = (53 * hash) + hashEnum(getState()); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -459,6 +535,8 @@ public final class ZooKeeperProtos { bitField0_ = (bitField0_ & ~0x00000001); rpcVersion_ = 0; bitField0_ = (bitField0_ & ~0x00000002); + state_ = org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State.OFFLINE; + bitField0_ = (bitField0_ & ~0x00000004); return this; } @@ -499,6 +577,10 @@ public final class ZooKeeperProtos { to_bitField0_ |= 0x00000002; } result.rpcVersion_ = rpcVersion_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.state_ = state_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -521,6 +603,9 @@ public final class ZooKeeperProtos { if (other.hasRpcVersion()) { setRpcVersion(other.getRpcVersion()); } + if (other.hasState()) { + setState(other.getState()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -564,7 +649,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * *
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        * 
*/ public boolean hasServer() { @@ -574,7 +660,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * *
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        * 
*/ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName getServer() { @@ -588,7 +675,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * *
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        * 
*/ public Builder setServer(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName value) { @@ -608,7 +696,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * *
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        * 
*/ public Builder setServer( @@ -626,7 +715,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * *
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        * 
*/ public Builder mergeServer(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName value) { @@ -649,7 +739,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * *
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        * 
*/ public Builder clearServer() { @@ -666,7 +757,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * *
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        * 
*/ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.Builder getServerBuilder() { @@ -678,7 +770,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * *
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        * 
*/ public org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerNameOrBuilder getServerOrBuilder() { @@ -692,7 +785,8 @@ public final class ZooKeeperProtos { * required .ServerName server = 1; * *
-       * The ServerName hosting the meta region currently.
+       * The ServerName hosting the meta region currently, or destination server,
+       * if meta region is in transition.
        * 
*/ private com.google.protobuf.SingleFieldBuilder< @@ -766,6 +860,58 @@ public final class ZooKeeperProtos { return this; } + // optional .RegionState.State state = 3; + private org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State state_ = org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State.OFFLINE; + /** + * optional .RegionState.State state = 3; + * + *
+       * State of the region transition. OPEN means fully operational 'hbase:meta'
+       * 
+ */ + public boolean hasState() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + /** + * optional .RegionState.State state = 3; + * + *
+       * State of the region transition. OPEN means fully operational 'hbase:meta'
+       * 
+ */ + public org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State getState() { + return state_; + } + /** + * optional .RegionState.State state = 3; + * + *
+       * State of the region transition. OPEN means fully operational 'hbase:meta'
+       * 
+ */ + public Builder setState(org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + state_ = value; + onChanged(); + return this; + } + /** + * optional .RegionState.State state = 3; + * + *
+       * State of the region transition. OPEN means fully operational 'hbase:meta'
+       * 
+ */ + public Builder clearState() { + bitField0_ = (bitField0_ & ~0x00000004); + state_ = org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State.OFFLINE; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:MetaRegionServer) } @@ -9414,39 +9560,40 @@ public final class ZooKeeperProtos { descriptor; static { java.lang.String[] descriptorData = { - "\n\017ZooKeeper.proto\032\013HBase.proto\"D\n\020MetaRe" + - "gionServer\022\033\n\006server\030\001 \002(\0132\013.ServerName\022" + - "\023\n\013rpc_version\030\002 \001(\r\":\n\006Master\022\033\n\006master" + - "\030\001 \002(\0132\013.ServerName\022\023\n\013rpc_version\030\002 \001(\r" + - "\"\037\n\tClusterUp\022\022\n\nstart_date\030\001 \002(\t\"\214\002\n\014Sp" + - "litLogTask\022\"\n\005state\030\001 \002(\0162\023.SplitLogTask" + - ".State\022 \n\013server_name\030\002 \002(\0132\013.ServerName" + - "\0221\n\004mode\030\003 \001(\0162\032.SplitLogTask.RecoveryMo" + - "de:\007UNKNOWN\"C\n\005State\022\016\n\nUNASSIGNED\020\000\022\t\n\005" + - "OWNED\020\001\022\014\n\010RESIGNED\020\002\022\010\n\004DONE\020\003\022\007\n\003ERR\020\004", - "\">\n\014RecoveryMode\022\013\n\007UNKNOWN\020\000\022\021\n\rLOG_SPL" + - "ITTING\020\001\022\016\n\nLOG_REPLAY\020\002\"n\n\005Table\022$\n\005sta" + - "te\030\001 \002(\0162\014.Table.State:\007ENABLED\"?\n\005State" + - "\022\013\n\007ENABLED\020\000\022\014\n\010DISABLED\020\001\022\r\n\tDISABLING" + - "\020\002\022\014\n\010ENABLING\020\003\"\215\001\n\017ReplicationPeer\022\022\n\n" + - "clusterkey\030\001 \002(\t\022\037\n\027replicationEndpointI" + - "mpl\030\002 \001(\t\022\035\n\004data\030\003 \003(\0132\017.BytesBytesPair" + - "\022&\n\rconfiguration\030\004 \003(\0132\017.NameStringPair" + - "\"^\n\020ReplicationState\022&\n\005state\030\001 \002(\0162\027.Re" + - "plicationState.State\"\"\n\005State\022\013\n\007ENABLED", - "\020\000\022\014\n\010DISABLED\020\001\"+\n\027ReplicationHLogPosit" + - "ion\022\020\n\010position\030\001 \002(\003\"%\n\017ReplicationLock" + - "\022\022\n\nlock_owner\030\001 \002(\t\"\230\001\n\tTableLock\022\036\n\nta" + - "ble_name\030\001 \001(\0132\n.TableName\022\037\n\nlock_owner" + - "\030\002 \001(\0132\013.ServerName\022\021\n\tthread_id\030\003 \001(\003\022\021" + - "\n\tis_shared\030\004 \001(\010\022\017\n\007purpose\030\005 \001(\t\022\023\n\013cr" + - "eate_time\030\006 \001(\003\";\n\017StoreSequenceId\022\023\n\013fa" + - "mily_name\030\001 \002(\014\022\023\n\013sequence_id\030\002 \002(\004\"g\n\026" + - "RegionStoreSequenceIds\022 \n\030last_flushed_s" + - "equence_id\030\001 \002(\004\022+\n\021store_sequence_id\030\002 ", - "\003(\0132\020.StoreSequenceIdBE\n*org.apache.hado" + - "op.hbase.protobuf.generatedB\017ZooKeeperPr" + - "otosH\001\210\001\001\240\001\001" + "\n\017ZooKeeper.proto\032\013HBase.proto\032\023ClusterS" + + "tatus.proto\"g\n\020MetaRegionServer\022\033\n\006serve" + + "r\030\001 \002(\0132\013.ServerName\022\023\n\013rpc_version\030\002 \001(" + + "\r\022!\n\005state\030\003 \001(\0162\022.RegionState.State\":\n\006" + + "Master\022\033\n\006master\030\001 \002(\0132\013.ServerName\022\023\n\013r" + + "pc_version\030\002 \001(\r\"\037\n\tClusterUp\022\022\n\nstart_d" + + "ate\030\001 \002(\t\"\214\002\n\014SplitLogTask\022\"\n\005state\030\001 \002(" + + "\0162\023.SplitLogTask.State\022 \n\013server_name\030\002 " + + "\002(\0132\013.ServerName\0221\n\004mode\030\003 \001(\0162\032.SplitLo" + + "gTask.RecoveryMode:\007UNKNOWN\"C\n\005State\022\016\n\n", + "UNASSIGNED\020\000\022\t\n\005OWNED\020\001\022\014\n\010RESIGNED\020\002\022\010\n" + + "\004DONE\020\003\022\007\n\003ERR\020\004\">\n\014RecoveryMode\022\013\n\007UNKN" + + "OWN\020\000\022\021\n\rLOG_SPLITTING\020\001\022\016\n\nLOG_REPLAY\020\002" + + "\"n\n\005Table\022$\n\005state\030\001 \002(\0162\014.Table.State:\007" + + "ENABLED\"?\n\005State\022\013\n\007ENABLED\020\000\022\014\n\010DISABLE" + + "D\020\001\022\r\n\tDISABLING\020\002\022\014\n\010ENABLING\020\003\"\215\001\n\017Rep" + + "licationPeer\022\022\n\nclusterkey\030\001 \002(\t\022\037\n\027repl" + + "icationEndpointImpl\030\002 \001(\t\022\035\n\004data\030\003 \003(\0132" + + "\017.BytesBytesPair\022&\n\rconfiguration\030\004 \003(\0132" + + "\017.NameStringPair\"^\n\020ReplicationState\022&\n\005", + "state\030\001 \002(\0162\027.ReplicationState.State\"\"\n\005" + + "State\022\013\n\007ENABLED\020\000\022\014\n\010DISABLED\020\001\"+\n\027Repl" + + "icationHLogPosition\022\020\n\010position\030\001 \002(\003\"%\n" + + "\017ReplicationLock\022\022\n\nlock_owner\030\001 \002(\t\"\230\001\n" + + "\tTableLock\022\036\n\ntable_name\030\001 \001(\0132\n.TableNa" + + "me\022\037\n\nlock_owner\030\002 \001(\0132\013.ServerName\022\021\n\tt" + + "hread_id\030\003 \001(\003\022\021\n\tis_shared\030\004 \001(\010\022\017\n\007pur" + + "pose\030\005 \001(\t\022\023\n\013create_time\030\006 \001(\003\";\n\017Store" + + "SequenceId\022\023\n\013family_name\030\001 \002(\014\022\023\n\013seque" + + "nce_id\030\002 \002(\004\"g\n\026RegionStoreSequenceIds\022 ", + "\n\030last_flushed_sequence_id\030\001 \002(\004\022+\n\021stor" + + "e_sequence_id\030\002 \003(\0132\020.StoreSequenceIdBE\n" + + "*org.apache.hadoop.hbase.protobuf.genera" + + "tedB\017ZooKeeperProtosH\001\210\001\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -9458,7 +9605,7 @@ public final class ZooKeeperProtos { internal_static_MetaRegionServer_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_MetaRegionServer_descriptor, - new java.lang.String[] { "Server", "RpcVersion", }); + new java.lang.String[] { "Server", "RpcVersion", "State", }); internal_static_Master_descriptor = getDescriptor().getMessageTypes().get(1); internal_static_Master_fieldAccessorTable = new @@ -9532,6 +9679,7 @@ public final class ZooKeeperProtos { .internalBuildGeneratedFileFrom(descriptorData, new com.google.protobuf.Descriptors.FileDescriptor[] { org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.getDescriptor(), + org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.getDescriptor(), }, assigner); } diff --git a/hbase-protocol/src/main/protobuf/ZooKeeper.proto b/hbase-protocol/src/main/protobuf/ZooKeeper.proto index 4d727c6..8acd778 100644 --- a/hbase-protocol/src/main/protobuf/ZooKeeper.proto +++ b/hbase-protocol/src/main/protobuf/ZooKeeper.proto @@ -26,17 +26,22 @@ option java_generate_equals_and_hash = true; option optimize_for = SPEED; import "HBase.proto"; +import "ClusterStatus.proto"; /** * Content of the meta-region-server znode. */ message MetaRegionServer { - // The ServerName hosting the meta region currently. + // The ServerName hosting the meta region currently, or destination server, + // if meta region is in transition. required ServerName server = 1; // The major version of the rpc the server speaks. This is used so that // clients connecting to the cluster can have prior knowledge of what version // to send to a RegionServer. AsyncHBase will use this to detect versions. optional uint32 rpc_version = 2; + + // State of the region transition. OPEN means fully operational 'hbase:meta' + optional RegionState.State state = 3; } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index eca5999..27ce9e0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -666,20 +666,26 @@ public class HMaster extends HRegionServer implements MasterServices, Server { RegionStates regionStates = assignmentManager.getRegionStates(); regionStates.createRegionState(HRegionInfo.FIRST_META_REGIONINFO); - boolean metaRegionLocation = metaTableLocator.verifyMetaRegionLocation( - this.getShortCircuitConnection(), this.getZooKeeper(), timeout); - ServerName currentMetaServer = metaTableLocator.getMetaRegionLocation(this.getZooKeeper()); - if (!metaRegionLocation) { - // Meta location is not verified. It should be in transition, or offline. - // We will wait for it to be assigned in enableSSHandWaitForMeta below. - if (currentMetaServer != null) { - // If the meta server is not known to be dead or online, - // just split the meta log, and don't expire it since this - // could be a full cluster restart. Otherwise, we will think - // this is a failover and lose previous region locations. - // If it is really a failover case, AM will find out in rebuilding - // user regions. Otherwise, we are good since all logs are split - // or known to be replayed before user regions are assigned. + + // get current meta location from zk. + MetaTableLocator.MetaRegionLocation currentMetaLocation = metaTableLocator + .getMetaRegionLocationAndState(this.getZooKeeper()); + ServerName currentMetaServer = currentMetaLocation.getServerName(); + if (currentMetaLocation.isOnline() + && metaTableLocator.verifyMetaRegionLocation( + this.getShortCircuitConnection(), this.getZooKeeper(), timeout)) { + // have open meta on alive server, assign it. + // Region already assigned. We didn't assign it. Add to in-memory state. + regionStates.updateRegionState( + HRegionInfo.FIRST_META_REGIONINFO, State.OPEN, currentMetaServer); + this.assignmentManager.regionOnline( + HRegionInfo.FIRST_META_REGIONINFO, currentMetaServer); + } else { + if(currentMetaLocation.isOffline()) { + // ok, meta was unassigned cleanly (or suppose to) + LOG.info("Meta is offline"); + } else { + LOG.info("Meta was in transition to " + currentMetaLocation); if (serverManager.isServerOnline(currentMetaServer)) { LOG.info("Forcing expire of " + currentMetaServer); serverManager.expireServer(currentMetaServer); @@ -689,12 +695,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } assignmentManager.assignMeta(); assigned++; - } else { - // Region already assigned. We didn't assign it. Add to in-memory state. - regionStates.updateRegionState( - HRegionInfo.FIRST_META_REGIONINFO, State.OPEN, currentMetaServer); - this.assignmentManager.regionOnline( - HRegionInfo.FIRST_META_REGIONINFO, currentMetaServer); } enableMeta(TableName.META_TABLE_NAME); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java index fb16dd3..7554475 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.master; import java.io.IOException; import java.util.Arrays; +import com.google.common.base.Preconditions; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -158,20 +159,31 @@ public class RegionStateStore { void updateRegionState(long openSeqNum, RegionState newState, RegionState oldState) { - if (!initialized) { - return; - } - HRegionInfo hri = newState.getRegion(); - if (!shouldPersistStateChange(hri, newState, oldState)) { - return; - } + try { + HRegionInfo hri = newState.getRegion(); + + // update meta before checking for initializaion. + // meta state stored in zk. + if (hri.isMetaRegion()) { + // persist meta state in MetaTableLocator (which in turn is zk storage currently) + server.getMetaTableLocator().setMetaLocation(server.getZooKeeper(), + newState.getServerName(), newState.getState()); + return; + } - ServerName oldServer = oldState != null ? oldState.getServerName() : null; - ServerName serverName = newState.getServerName(); - State state = newState.getState(); + if (!initialized) { + return; + } + + if (!shouldPersistStateChange(hri, newState, oldState)) { + return; + } + + ServerName oldServer = oldState != null ? oldState.getServerName() : null; + ServerName serverName = newState.getServerName(); + State state = newState.getState(); - try { int replicaId = hri.getReplicaId(); Put put = new Put(MetaTableAccessor.getMetaKeyForRegion(hri)); StringBuilder info = new StringBuilder("Updating row "); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 8e62620..393a5e0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -1719,10 +1719,6 @@ public class HRegionServer extends HasThread implements // Update flushed sequence id of a recovering region in ZK updateRecoveringRegionLastFlushedSequenceId(r); - if (r.getRegionInfo().isMetaRegion()) { - MetaTableLocator.setMetaLocation(getZooKeeper(), serverName); - } - // Notify master if (!reportRegionStateTransition( TransitionCode.OPENED, openSeqNum, r.getRegionInfo())) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java index dccd7ca..0d1d2af 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableLocator.java @@ -18,6 +18,8 @@ */ package org.apache.hadoop.hbase; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.io.IOException; @@ -32,6 +34,7 @@ import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.HConnectionTestingUtility; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; @@ -106,6 +109,37 @@ public class TestMetaTableLocator { } /** + * Test normal operations + */ + @Test public void testMetaLookup() + throws IOException, InterruptedException, ServiceException, KeeperException { + final ClientProtos.ClientService.BlockingInterface client = + Mockito.mock(ClientProtos.ClientService.BlockingInterface.class); + + Mockito.when(client.get((RpcController)Mockito.any(), (GetRequest)Mockito.any())). + thenReturn(GetResponse.newBuilder().build()); + + final MetaTableLocator mtl = new MetaTableLocator(); + assertNull(mtl.getMetaRegionLocation(this.watcher)); + for (RegionState.State state : RegionState.State.values()) { + if (state.equals(RegionState.State.OPEN)) + continue; + mtl.setMetaLocation(this.watcher, SN, state); + assertNull(mtl.getMetaRegionLocation(this.watcher)); + assertEquals(state, mtl.getMetaRegionLocationAndState(this.watcher).getState()); + } + mtl.setMetaLocation(this.watcher, SN, RegionState.State.OPEN); + assertEquals(mtl.getMetaRegionLocation(this.watcher), SN); + assertEquals(RegionState.State.OPEN, + mtl.getMetaRegionLocationAndState(this.watcher).getState()); + + mtl.deleteMetaLocation(this.watcher); + assertNull(mtl.getMetaRegionLocation(this.watcher)); + assertNull(mtl.getMetaRegionLocationAndState(this.watcher)); + } + + + /** * Test interruptable while blocking wait on meta. * @throws IOException * @throws ServiceException @@ -153,11 +187,15 @@ public class TestMetaTableLocator { Mockito.when(implementation.get((RpcController) Mockito.any(), (GetRequest) Mockito.any())). thenThrow(new ServiceException(ex)); - MetaTableLocator.setMetaLocation(this.watcher, SN); long timeout = UTIL.getConfiguration(). - getLong("hbase.catalog.verification.timeout", 1000); + getLong("hbase.catalog.verification.timeout", 1000); + new MetaTableLocator().setMetaLocation(this.watcher, SN, RegionState.State.OPENING); Assert.assertFalse(new MetaTableLocator().verifyMetaRegionLocation( connection, watcher, timeout)); + + new MetaTableLocator().setMetaLocation(this.watcher, SN, RegionState.State.OPEN); + Assert.assertFalse(new MetaTableLocator().verifyMetaRegionLocation( + connection, watcher, timeout)); } /** @@ -213,8 +251,14 @@ public class TestMetaTableLocator { Mockito.when(connection.getAdmin(Mockito.any(ServerName.class), Mockito.anyBoolean())). thenReturn(implementation); - MetaTableLocator.setMetaLocation(this.watcher, - ServerName.valueOf("example.com", 1234, System.currentTimeMillis())); + ServerName sn = ServerName.valueOf("example.com", 1234, System.currentTimeMillis()); + new MetaTableLocator().setMetaLocation(this.watcher, + sn, + RegionState.State.OPENING); + Assert.assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100)); + new MetaTableLocator().setMetaLocation(this.watcher, + sn, + RegionState.State.OPEN); Assert.assertFalse(new MetaTableLocator().verifyMetaRegionLocation(connection, watcher, 100)); } @@ -240,18 +284,14 @@ public class TestMetaTableLocator { Thread t = new WaitOnMetaThread(); startWaitAliveThenWaitItLives(t, 1); // Set a meta location. - hsa = setMetaLocation(); + new MetaTableLocator().setMetaLocation(this.watcher, SN, RegionState.State.OPEN); + hsa = SN; // Join the thread... should exit shortly. t.join(); // Now meta is available. Assert.assertTrue(mtl.getMetaRegionLocation(watcher).equals(hsa)); } - private ServerName setMetaLocation() throws KeeperException { - MetaTableLocator.setMetaLocation(this.watcher, SN); - return SN; - } - /** * @param admin An {@link AdminProtos.AdminService.BlockingInterface} instance; you'll likely * want to pass a mocked HRS; can be null. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java index 02847d1..9bf4655 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManagerOnCluster.java @@ -17,14 +17,6 @@ */ package org.apache.hadoop.hbase.master; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -68,12 +60,21 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.zookeeper.KeeperException; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + /** * This tests AssignmentManager with a testing cluster. */ @@ -120,11 +121,14 @@ public class TestAssignmentManagerOnCluster { metaServerName = cluster.getLiveRegionServerThreads() .get(0).getRegionServer().getServerName(); master.move(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), - Bytes.toBytes(metaServerName.getServerName())); + Bytes.toBytes(metaServerName.getServerName())); TEST_UTIL.waitUntilNoRegionsInTransition(60000); } + MetaTableLocator.MetaRegionLocation rl = master.getMetaTableLocator() + .getMetaRegionLocationAndState(master.getZooKeeper()); + assertEquals("Meta should be not in transition", rl.getState(), State.OPEN); assertNotEquals("Meta should be moved off master", - metaServerName, master.getServerName()); + rl.getServerName(), master.getServerName()); cluster.killRegionServer(metaServerName); stoppedARegionServer = true; cluster.waitForRegionServerToStop(metaServerName, 60000); @@ -138,9 +142,15 @@ public class TestAssignmentManagerOnCluster { } }); + TEST_UTIL.waitUntilNoRegionsInTransition(60000); // Now, make sure meta is assigned assertTrue("Meta should be assigned", regionStates.isRegionOnline(HRegionInfo.FIRST_META_REGIONINFO)); + // Now, make sure meta is registered in zk + rl = master.getMetaTableLocator() + .getMetaRegionLocationAndState(master.getZooKeeper()); + assertEquals("Meta should be not in transition", rl.getState(), State.OPEN); + assertEquals("Meta should be assigned", master.getServerName(), rl.getServerName()); } finally { if (stoppedARegionServer) { cluster.startRegionServer(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java index 5c35611..063ecd2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java @@ -18,10 +18,6 @@ */ package org.apache.hadoop.hbase.master; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - import java.io.IOException; import java.util.List; @@ -45,13 +41,19 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; +import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.junit.Test; import org.junit.experimental.categories.Category; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + @Category(LargeTests.class) public class TestMasterFailover { private static final Log LOG = LogFactory.getLog(TestMasterFailover.class); @@ -281,5 +283,121 @@ public class TestMasterFailover { // Done, shutdown the cluster TEST_UTIL.shutdownMiniCluster(); } + + /** + * Test meta in transition when master failover + */ + @Test(timeout = 180000) + public void testMetaInTransitionWhenMasterFailover() throws Exception { + final int NUM_MASTERS = 1; + final int NUM_RS = 1; + + // Start the cluster + HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + log("Cluster started"); + + log("Moving meta off the master"); + HMaster activeMaster = cluster.getMaster(); + HRegionServer rs = cluster.getRegionServer(0); + ServerName metaServerName = cluster.getLiveRegionServerThreads() + .get(0).getRegionServer().getServerName(); + activeMaster.move(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), + Bytes.toBytes(metaServerName.getServerName())); + TEST_UTIL.waitUntilNoRegionsInTransition(60000); + TEST_UTIL.waitUntilNoRegionsInTransition(60000); + + + // Now kill master, meta should remain on rs, where we placed it before. + log("Aborting master"); + activeMaster.abort("test-kill"); + cluster.waitForMasterToStop(activeMaster.getServerName(), 30000); + log("Master has aborted"); + + // meta should remain where it was + MetaTableLocator.MetaRegionLocation metaLocation = + activeMaster.getMetaTableLocator().getMetaRegionLocationAndState(rs.getZooKeeper()); + assertEquals("hbase:meta should be onlined on RS", + metaLocation.getServerName(), rs.getServerName()); + assertEquals("hbase:meta should be onlined on RS", + metaLocation.getState(), State.OPEN); + + // Start up a new master + log("Starting up a new master"); + activeMaster = cluster.startMaster().getMaster(); + log("Waiting for master to be ready"); + cluster.waitForActiveAndReadyMaster(); + log("Master is ready"); + + // ensure meta is still deployed on RS + metaLocation = + activeMaster.getMetaTableLocator().getMetaRegionLocationAndState(activeMaster.getZooKeeper()); + assertEquals("hbase:meta should be onlined on RS", + metaLocation.getServerName(), rs.getServerName()); + assertEquals("hbase:meta should be onlined on RS", + metaLocation.getState(), State.OPEN); + + // update meta state as in transition, then kill master + // that simulates, that RS successfully deployed, but + // RPC was lost right before failure. + // region server should expire (how it can be verified?) + activeMaster.getMetaTableLocator().setMetaLocation(activeMaster.getZooKeeper(), + rs.getServerName(), State.PENDING_OPEN); + log("Aborting master"); + activeMaster.abort("test-kill"); + cluster.waitForMasterToStop(activeMaster.getServerName(), 30000); + log("Master has aborted"); + + // Start up a new master + log("Starting up a new master"); + activeMaster = cluster.startMaster().getMaster(); + log("Waiting for master to be ready"); + cluster.waitForActiveAndReadyMaster(); + log("Master is ready"); + + TEST_UTIL.waitUntilNoRegionsInTransition(60000); + log("Meta was assigned"); + + // wait for rs be expired + cluster.waitForRegionServerToStop(rs.getServerName(), 60000); + + // meta should be moved onto active master + metaLocation = + activeMaster.getMetaTableLocator().getMetaRegionLocationAndState(activeMaster.getZooKeeper()); + assertEquals("hbase:meta should be onlined on RS", + metaLocation.getServerName(), activeMaster.getServerName()); + assertEquals("hbase:meta should be onlined on RS", + metaLocation.getState(), State.OPEN); + + // place meta on master and kill it + // new master should successfully reassign it to itself + + log("Done with verification, shutting down cluster"); + log("Aborting master"); + activeMaster.abort("test-kill"); + cluster.waitForMasterToStop(activeMaster.getServerName(), 30000); + log("Master has aborted"); + + // Start up a new master + log("Starting up a new master"); + activeMaster = cluster.startMaster().getMaster(); + log("Waiting for master to be ready"); + cluster.waitForActiveAndReadyMaster(); + log("Master is ready"); + + TEST_UTIL.waitUntilNoRegionsInTransition(60000); + log("Meta was assigned"); + // meta should be moved onto active master + metaLocation = + activeMaster.getMetaTableLocator().getMetaRegionLocationAndState(activeMaster.getZooKeeper()); + assertEquals("hbase:meta should be onlined on RS", + metaLocation.getServerName(), activeMaster.getServerName()); + assertEquals("hbase:meta should be onlined on RS", + metaLocation.getState(), State.OPEN); + + // Done, shutdown the cluster + TEST_UTIL.shutdownMiniCluster(); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java index 9f18f87..d1b7b35 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java @@ -157,7 +157,8 @@ public class TestMasterNoCluster { final MockRegionServer rs2 = new MockRegionServer(conf, sn2); // Put some data into the servers. Make it look like sn0 has the metaH // Put data into sn2 so it looks like it has a few regions for a table named 't'. - MetaTableLocator.setMetaLocation(rs0.getZooKeeper(), rs0.getServerName()); + new MetaTableLocator().setMetaLocation(rs0.getZooKeeper(), + rs0.getServerName(), RegionState.State.OPEN); final TableName tableName = TableName.valueOf("t"); Result [] results = new Result [] { MetaMockingUtil.getMetaTableRowResult(