commit f105ecf42899f13ce6318820e81603062d2ec607 Author: Virag Kothari Date: Fri Nov 14 14:43:35 2014 -0800 HBASE-12479 Backport HBASE-11689 (Track meta in transition) to 0.98 and branch-1 diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java index 1755240..47a2b7d 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ServerName.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.MetaRegionServer; import org.apache.hadoop.hbase.util.Addressing; import org.apache.hadoop.hbase.util.Bytes; @@ -54,6 +55,8 @@ import java.util.regex.Pattern; @InterfaceAudience.Public @InterfaceStability.Evolving public class ServerName implements Comparable, Serializable { + private static final long serialVersionUID = 1367463982557264981L; + /** * Version for this class. * Its a short rather than a byte so I can for sure distinguish between this @@ -370,9 +373,9 @@ public class ServerName implements Comparable, Serializable { if (ProtobufUtil.isPBMagicPrefix(data)) { int prefixLen = ProtobufUtil.lengthOfPBMagic(); try { - MetaRegionServer rss = - MetaRegionServer.PARSER.parseFrom(data, prefixLen, data.length - prefixLen); - org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName sn = rss.getServer(); + ZooKeeperProtos.Master rss = + ZooKeeperProtos.Master.PARSER.parseFrom(data, prefixLen, data.length - prefixLen); + org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName sn = rss.getMaster(); return valueOf(sn.getHostName(), sn.getPort(), sn.getStartCode()); } catch (InvalidProtocolBufferException e) { // A failed parse of the znode is pretty catastrophic. Rather than loop diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java index 09c66cc..08b7ae7 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/CatalogTracker.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.RetriesExhaustedException; import org.apache.hadoop.hbase.ipc.RpcClient.FailedServerException; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.regionserver.RegionServerStoppedException; @@ -273,6 +274,16 @@ public class CatalogTracker { } return sn; } + + /** + * Get meta region state + * @return RegionState + */ + public RegionState getMetaRegionState() { + return metaRegionTracker.getMetaRegionState(); + } + + /** * Gets a connection to the server hosting meta, as reported by ZooKeeper, diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java index 6ee796a..ec96196 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/master/RegionState.java @@ -53,9 +53,125 @@ public class RegionState { SPLITTING_NEW, // new region to be created when RS splits a parent // region but hasn't be created yet, or master doesn't // know it's already created - MERGING_NEW // new region to be created when RS merges two + MERGING_NEW; // new region to be created when RS merges two // daughter regions but hasn't be created yet, or // master doesn't know it's already created + + /** + * Convert to protobuf ClusterStatusProtos.RegionState.State + */ + public ClusterStatusProtos.RegionState.State convert() { + ClusterStatusProtos.RegionState.State rs; + switch (this) { + case OFFLINE: + rs = ClusterStatusProtos.RegionState.State.OFFLINE; + break; + case PENDING_OPEN: + rs = ClusterStatusProtos.RegionState.State.PENDING_OPEN; + break; + case OPENING: + rs = ClusterStatusProtos.RegionState.State.OPENING; + break; + case OPEN: + rs = ClusterStatusProtos.RegionState.State.OPEN; + break; + case PENDING_CLOSE: + rs = ClusterStatusProtos.RegionState.State.PENDING_CLOSE; + break; + case CLOSING: + rs = ClusterStatusProtos.RegionState.State.CLOSING; + break; + case CLOSED: + rs = ClusterStatusProtos.RegionState.State.CLOSED; + break; + case SPLITTING: + rs = ClusterStatusProtos.RegionState.State.SPLITTING; + break; + case SPLIT: + rs = ClusterStatusProtos.RegionState.State.SPLIT; + break; + case FAILED_OPEN: + rs = ClusterStatusProtos.RegionState.State.FAILED_OPEN; + break; + case FAILED_CLOSE: + rs = ClusterStatusProtos.RegionState.State.FAILED_CLOSE; + break; + case MERGING: + rs = ClusterStatusProtos.RegionState.State.MERGING; + break; + case MERGED: + rs = ClusterStatusProtos.RegionState.State.MERGED; + break; + case SPLITTING_NEW: + rs = ClusterStatusProtos.RegionState.State.SPLITTING_NEW; + break; + case MERGING_NEW: + rs = ClusterStatusProtos.RegionState.State.MERGING_NEW; + break; + default: + throw new IllegalStateException(""); + } + return rs; + } + + /** + * Convert a protobuf HBaseProtos.RegionState.State to a RegionState.State + * + * @return the RegionState.State + */ + public static State convert(ClusterStatusProtos.RegionState.State protoState) { + State state; + switch (protoState) { + case OFFLINE: + state = OFFLINE; + break; + case PENDING_OPEN: + state = PENDING_OPEN; + break; + case OPENING: + state = OPENING; + break; + case OPEN: + state = OPEN; + break; + case PENDING_CLOSE: + state = PENDING_CLOSE; + break; + case CLOSING: + state = CLOSING; + break; + case CLOSED: + state = CLOSED; + break; + case SPLITTING: + state = SPLITTING; + break; + case SPLIT: + state = SPLIT; + break; + case FAILED_OPEN: + state = FAILED_OPEN; + break; + case FAILED_CLOSE: + state = FAILED_CLOSE; + break; + case MERGING: + state = MERGING; + break; + case MERGED: + state = MERGED; + break; + case SPLITTING_NEW: + state = SPLITTING_NEW; + break; + case MERGING_NEW: + state = MERGING_NEW; + break; + default: + throw new IllegalStateException(""); + } + return state; + } } // Many threads can update the state at the stamp at the same time @@ -264,58 +380,8 @@ public class RegionState { */ public ClusterStatusProtos.RegionState convert() { ClusterStatusProtos.RegionState.Builder regionState = ClusterStatusProtos.RegionState.newBuilder(); - ClusterStatusProtos.RegionState.State rs; - switch (this.state) { - case OFFLINE: - rs = ClusterStatusProtos.RegionState.State.OFFLINE; - break; - case PENDING_OPEN: - rs = ClusterStatusProtos.RegionState.State.PENDING_OPEN; - break; - case OPENING: - rs = ClusterStatusProtos.RegionState.State.OPENING; - break; - case OPEN: - rs = ClusterStatusProtos.RegionState.State.OPEN; - break; - case PENDING_CLOSE: - rs = ClusterStatusProtos.RegionState.State.PENDING_CLOSE; - break; - case CLOSING: - rs = ClusterStatusProtos.RegionState.State.CLOSING; - break; - case CLOSED: - rs = ClusterStatusProtos.RegionState.State.CLOSED; - break; - case SPLITTING: - rs = ClusterStatusProtos.RegionState.State.SPLITTING; - break; - case SPLIT: - rs = ClusterStatusProtos.RegionState.State.SPLIT; - break; - case FAILED_OPEN: - rs = ClusterStatusProtos.RegionState.State.FAILED_OPEN; - break; - case FAILED_CLOSE: - rs = ClusterStatusProtos.RegionState.State.FAILED_CLOSE; - break; - case MERGING: - rs = ClusterStatusProtos.RegionState.State.MERGING; - break; - case MERGED: - rs = ClusterStatusProtos.RegionState.State.MERGED; - break; - case SPLITTING_NEW: - rs = ClusterStatusProtos.RegionState.State.SPLITTING_NEW; - break; - case MERGING_NEW: - rs = ClusterStatusProtos.RegionState.State.MERGING_NEW; - break; - default: - throw new IllegalStateException(""); - } regionState.setRegionInfo(HRegionInfo.convert(hri)); - regionState.setState(rs); + regionState.setState(state.convert()); regionState.setStamp(getStamp()); return regionState.build(); } @@ -326,58 +392,8 @@ public class RegionState { * @return the RegionState */ public static RegionState convert(ClusterStatusProtos.RegionState proto) { - RegionState.State state; - switch (proto.getState()) { - case OFFLINE: - state = State.OFFLINE; - break; - case PENDING_OPEN: - state = State.PENDING_OPEN; - break; - case OPENING: - state = State.OPENING; - break; - case OPEN: - state = State.OPEN; - break; - case PENDING_CLOSE: - state = State.PENDING_CLOSE; - break; - case CLOSING: - state = State.CLOSING; - break; - case CLOSED: - state = State.CLOSED; - break; - case SPLITTING: - state = State.SPLITTING; - break; - case SPLIT: - state = State.SPLIT; - break; - case FAILED_OPEN: - state = State.FAILED_OPEN; - break; - case FAILED_CLOSE: - state = State.FAILED_CLOSE; - break; - case MERGING: - state = State.MERGING; - break; - case MERGED: - state = State.MERGED; - break; - case SPLITTING_NEW: - state = State.SPLITTING_NEW; - break; - case MERGING_NEW: - state = State.MERGING_NEW; - break; - default: - throw new IllegalStateException(""); - } - - return new RegionState(HRegionInfo.convert(proto.getRegionInfo()),state,proto.getStamp(),null); + return new RegionState(HRegionInfo.convert(proto.getRegionInfo()), + State.convert(proto.getState()), proto.getStamp(), null); } protected void setTimestamp(final long timestamp) { diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaRegionTracker.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaRegionTracker.java index 70f8156..5d106cf 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaRegionTracker.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/MetaRegionTracker.java @@ -20,13 +20,18 @@ package org.apache.hadoop.hbase.zookeeper; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos; import org.apache.zookeeper.KeeperException; +import com.google.common.base.Stopwatch; +import com.google.protobuf.InvalidProtocolBufferException; + /** * Tracks the meta region server location node in zookeeper. * Meta region location is set by RegionServerServices. @@ -51,7 +56,7 @@ public class MetaRegionTracker extends ZooKeeperNodeTracker { * @return true if meta region location is available, false if not */ public boolean isLocationAvailable() { - return super.getData(true) != null; + return getMetaRegionLocation() != null; } /** @@ -59,13 +64,9 @@ public class MetaRegionTracker extends ZooKeeperNodeTracker { * @return server name or null if we failed to get the data. * @throws InterruptedException */ - public ServerName getMetaRegionLocation() throws InterruptedException { - try { - return ServerName.parseFrom(super.getData(true)); - } catch (DeserializationException e) { - LOG.warn("Failed parse", e); - return null; - } + public ServerName getMetaRegionLocation() { + RegionState regionState = getMetaRegionState(); + return regionState.isOpened() ? regionState.getServerName() : null; } /** @@ -75,13 +76,10 @@ public class MetaRegionTracker extends ZooKeeperNodeTracker { * @return server name or null if we failed to get the data. * @throws KeeperException */ - public static ServerName getMetaRegionLocation(final ZooKeeperWatcher zkw) - throws KeeperException { - try { - return ServerName.parseFrom(ZKUtil.getData(zkw, zkw.metaServerZNode)); - } catch (DeserializationException e) { - throw ZKUtil.convert(e); - } + public static ServerName getMetaRegionLocation(final ZooKeeperWatcher zkw) throws KeeperException { + byte[] data = ZKUtil.getData(zkw, zkw.metaServerZNode); + RegionState regionState = getMetaRegionState(data); + return regionState.isOpened() ? regionState.getServerName() : null; } /** @@ -102,12 +100,33 @@ public class MetaRegionTracker extends ZooKeeperNodeTracker { LOG.error(errorMsg); throw new IllegalArgumentException(errorMsg); } + return blockUntilAvailable(timeout); + + } + + /** + * Wait until the meta region is available and is not in transition. + * @param zkw zookeeper connection to use + * @param timeout maximum time to wait, in millis + * @return ServerName or null if we timed out. + * @throws InterruptedException + */ + private ServerName blockUntilAvailable(final long timeout) throws InterruptedException { + if (timeout < 0) throw new IllegalArgumentException("Timeout shouldn't be less than zero"); + Stopwatch sw = new Stopwatch().start(); + ServerName sn = null; try { - return ServerName.parseFrom(super.blockUntilAvailable(timeout, true)); - } catch (DeserializationException e) { - LOG.warn("Failed parse", e); - return null; + while (true) { + sn = getMetaRegionLocation(); + if (sn != null || sw.elapsedMillis() > timeout - HConstants.SOCKET_RETRY_WAIT_MS) { + break; + } + Thread.sleep(HConstants.SOCKET_RETRY_WAIT_MS); + } + } finally { + sw.stop(); } + return sn; } /** @@ -118,12 +137,12 @@ public class MetaRegionTracker extends ZooKeeperNodeTracker { * @throws KeeperException unexpected zookeeper exception */ public static void setMetaLocation(ZooKeeperWatcher zookeeper, - final ServerName location) + final ServerName location, final RegionState.State regionState) throws KeeperException { LOG.info("Setting hbase:meta region location in ZooKeeper as " + location); // Make the MetaRegionServer pb and then get its bytes and save this as // the znode content. - byte [] data = toByteArray(location); + byte [] data = toByteArray(location, regionState); try { ZKUtil.createAndWatch(zookeeper, zookeeper.metaServerZNode, data); } catch(KeeperException.NodeExistsException nee) { @@ -137,7 +156,7 @@ public class MetaRegionTracker extends ZooKeeperNodeTracker { * @param sn What to put into the znode. * @return The content of the meta-region-server znode */ - static byte [] toByteArray(final ServerName sn) { + static byte [] toByteArray(final ServerName sn, final RegionState.State regionState) { // ZNode content is a pb message preceded by some pb magic. HBaseProtos.ServerName pbsn = HBaseProtos.ServerName.newBuilder() @@ -150,6 +169,7 @@ public class MetaRegionTracker extends ZooKeeperNodeTracker { ZooKeeperProtos.MetaRegionServer.newBuilder() .setServer(pbsn) .setRpcVersion(HConstants.RPC_CURRENT_VERSION) + .setState(regionState.convert()) .build(); return ProtobufUtil.prependPBMagic(pbrsr.toByteArray()); } @@ -181,12 +201,49 @@ public class MetaRegionTracker extends ZooKeeperNodeTracker { final long timeout) throws InterruptedException { byte [] data = ZKUtil.blockUntilAvailable(zkw, zkw.metaServerZNode, timeout); - if (data == null) return null; + RegionState regionState = getMetaRegionState(data); + return regionState.isOpened() ? regionState.getServerName() : null; + } + + + private static RegionState getMetaRegionState(byte[] data) { + RegionState.State state = RegionState.State.OPEN; + ServerName serverName = null; try { - return ServerName.parseFrom(data); + if (data != null && data.length > 0 && ProtobufUtil.isPBMagicPrefix(data)) { + try { + int prefixLen = ProtobufUtil.lengthOfPBMagic(); + ZooKeeperProtos.MetaRegionServer rl = + ZooKeeperProtos.MetaRegionServer.PARSER.parseFrom(data, prefixLen, data.length + - prefixLen); + if (rl.hasState()) { + state = RegionState.State.convert(rl.getState()); + } + HBaseProtos.ServerName sn = rl.getServer(); + serverName = ServerName.valueOf(sn.getHostName(), sn.getPort(), sn.getStartCode()); + } catch (InvalidProtocolBufferException e) { + throw new DeserializationException("Unable to parse root region location"); + } + } else { + // old style of meta region location? + serverName = ServerName.parseFrom(data); + } } catch (DeserializationException e) { LOG.warn("Failed parse", e); return null; } + if (serverName == null) { + state = RegionState.State.OFFLINE; + } + return new RegionState(HRegionInfo.FIRST_META_REGIONINFO, state, serverName); + } + + /** + * Get meta region state + * @return RegionState + */ + public RegionState getMetaRegionState() { + byte[] data = super.getData(true); + return getMetaRegionState(data); } -} +} \ No newline at end of file diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java index 9d037f5..2c7fcb0 100644 --- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ZooKeeperProtos.java @@ -58,6 +58,24 @@ public final class ZooKeeperProtos { * */ int getRpcVersion(); + + // optional .RegionState.State state = 3; + /** + * optional .RegionState.State state = 3; + * + *
+     * State of the region transition. OPEN means fully operational 'hbase:meta'
+     * 
+ */ + boolean hasState(); + /** + * optional .RegionState.State state = 3; + * + *
+     * State of the region transition. OPEN means fully operational 'hbase:meta'
+     * 
+ */ + org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State getState(); } /** * Protobuf type {@code MetaRegionServer} @@ -133,6 +151,17 @@ public final class ZooKeeperProtos { rpcVersion_ = input.readUInt32(); break; } + case 24: { + int rawValue = input.readEnum(); + org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State value = org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State.valueOf(rawValue); + if (value == null) { + unknownFields.mergeVarintField(3, rawValue); + } else { + bitField0_ |= 0x00000004; + state_ = value; + } + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -235,9 +264,34 @@ public final class ZooKeeperProtos { return rpcVersion_; } + // optional .RegionState.State state = 3; + public static final int STATE_FIELD_NUMBER = 3; + private org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State state_; + /** + * optional .RegionState.State state = 3; + * + *
+     * State of the region transition. OPEN means fully operational 'hbase:meta'
+     * 
+ */ + public boolean hasState() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + /** + * optional .RegionState.State state = 3; + * + *
+     * State of the region transition. OPEN means fully operational 'hbase:meta'
+     * 
+ */ + public org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State getState() { + return state_; + } + private void initFields() { server_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ServerName.getDefaultInstance(); rpcVersion_ = 0; + state_ = org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State.OFFLINE; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -265,6 +319,9 @@ public final class ZooKeeperProtos { if (((bitField0_ & 0x00000002) == 0x00000002)) { output.writeUInt32(2, rpcVersion_); } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeEnum(3, state_.getNumber()); + } getUnknownFields().writeTo(output); } @@ -282,6 +339,10 @@ public final class ZooKeeperProtos { size += com.google.protobuf.CodedOutputStream .computeUInt32Size(2, rpcVersion_); } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream + .computeEnumSize(3, state_.getNumber()); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -315,6 +376,11 @@ public final class ZooKeeperProtos { result = result && (getRpcVersion() == other.getRpcVersion()); } + result = result && (hasState() == other.hasState()); + if (hasState()) { + result = result && + (getState() == other.getState()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -336,6 +402,10 @@ public final class ZooKeeperProtos { hash = (37 * hash) + RPC_VERSION_FIELD_NUMBER; hash = (53 * hash) + getRpcVersion(); } + if (hasState()) { + hash = (37 * hash) + STATE_FIELD_NUMBER; + hash = (53 * hash) + hashEnum(getState()); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -459,6 +529,8 @@ public final class ZooKeeperProtos { bitField0_ = (bitField0_ & ~0x00000001); rpcVersion_ = 0; bitField0_ = (bitField0_ & ~0x00000002); + state_ = org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State.OFFLINE; + bitField0_ = (bitField0_ & ~0x00000004); return this; } @@ -499,6 +571,10 @@ public final class ZooKeeperProtos { to_bitField0_ |= 0x00000002; } result.rpcVersion_ = rpcVersion_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.state_ = state_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -521,6 +597,9 @@ public final class ZooKeeperProtos { if (other.hasRpcVersion()) { setRpcVersion(other.getRpcVersion()); } + if (other.hasState()) { + setState(other.getState()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -766,6 +845,58 @@ public final class ZooKeeperProtos { return this; } + // optional .RegionState.State state = 3; + private org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State state_ = org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State.OFFLINE; + /** + * optional .RegionState.State state = 3; + * + *
+       * State of the region transition. OPEN means fully operational 'hbase:meta'
+       * 
+ */ + public boolean hasState() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + /** + * optional .RegionState.State state = 3; + * + *
+       * State of the region transition. OPEN means fully operational 'hbase:meta'
+       * 
+ */ + public org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State getState() { + return state_; + } + /** + * optional .RegionState.State state = 3; + * + *
+       * State of the region transition. OPEN means fully operational 'hbase:meta'
+       * 
+ */ + public Builder setState(org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + state_ = value; + onChanged(); + return this; + } + /** + * optional .RegionState.State state = 3; + * + *
+       * State of the region transition. OPEN means fully operational 'hbase:meta'
+       * 
+ */ + public Builder clearState() { + bitField0_ = (bitField0_ & ~0x00000004); + state_ = org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.RegionState.State.OFFLINE; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:MetaRegionServer) } @@ -9581,40 +9712,41 @@ public final class ZooKeeperProtos { descriptor; static { java.lang.String[] descriptorData = { - "\n\017ZooKeeper.proto\032\013HBase.proto\"D\n\020MetaRe" + - "gionServer\022\033\n\006server\030\001 \002(\0132\013.ServerName\022" + - "\023\n\013rpc_version\030\002 \001(\r\":\n\006Master\022\033\n\006master" + - "\030\001 \002(\0132\013.ServerName\022\023\n\013rpc_version\030\002 \001(\r" + - "\"\037\n\tClusterUp\022\022\n\nstart_date\030\001 \002(\t\"\210\001\n\020Re" + - "gionTransition\022\027\n\017event_type_code\030\001 \002(\r\022" + - "\023\n\013region_name\030\002 \002(\014\022\023\n\013create_time\030\003 \002(" + - "\004\022 \n\013server_name\030\004 \002(\0132\013.ServerName\022\017\n\007p" + - "ayload\030\005 \001(\014\"\214\002\n\014SplitLogTask\022\"\n\005state\030\001" + - " \002(\0162\023.SplitLogTask.State\022 \n\013server_name", - "\030\002 \002(\0132\013.ServerName\0221\n\004mode\030\003 \001(\0162\032.Spli" + - "tLogTask.RecoveryMode:\007UNKNOWN\"C\n\005State\022" + - "\016\n\nUNASSIGNED\020\000\022\t\n\005OWNED\020\001\022\014\n\010RESIGNED\020\002" + - "\022\010\n\004DONE\020\003\022\007\n\003ERR\020\004\">\n\014RecoveryMode\022\013\n\007U" + - "NKNOWN\020\000\022\021\n\rLOG_SPLITTING\020\001\022\016\n\nLOG_REPLA" + - "Y\020\002\"n\n\005Table\022$\n\005state\030\001 \002(\0162\014.Table.Stat" + - "e:\007ENABLED\"?\n\005State\022\013\n\007ENABLED\020\000\022\014\n\010DISA" + - "BLED\020\001\022\r\n\tDISABLING\020\002\022\014\n\010ENABLING\020\003\"%\n\017R" + - "eplicationPeer\022\022\n\nclusterkey\030\001 \002(\t\"^\n\020Re" + - "plicationState\022&\n\005state\030\001 \002(\0162\027.Replicat", - "ionState.State\"\"\n\005State\022\013\n\007ENABLED\020\000\022\014\n\010" + - "DISABLED\020\001\"+\n\027ReplicationHLogPosition\022\020\n" + - "\010position\030\001 \002(\003\"%\n\017ReplicationLock\022\022\n\nlo" + - "ck_owner\030\001 \002(\t\"\230\001\n\tTableLock\022\036\n\ntable_na" + - "me\030\001 \001(\0132\n.TableName\022\037\n\nlock_owner\030\002 \001(\013" + - "2\013.ServerName\022\021\n\tthread_id\030\003 \001(\003\022\021\n\tis_s" + - "hared\030\004 \001(\010\022\017\n\007purpose\030\005 \001(\t\022\023\n\013create_t" + - "ime\030\006 \001(\003\";\n\017StoreSequenceId\022\023\n\013family_n" + - "ame\030\001 \002(\014\022\023\n\013sequence_id\030\002 \002(\004\"g\n\026Region" + - "StoreSequenceIds\022 \n\030last_flushed_sequenc", - "e_id\030\001 \002(\004\022+\n\021store_sequence_id\030\002 \003(\0132\020." + - "StoreSequenceIdBE\n*org.apache.hadoop.hba" + - "se.protobuf.generatedB\017ZooKeeperProtosH\001" + - "\210\001\001\240\001\001" + "\n\017ZooKeeper.proto\032\013HBase.proto\032\023ClusterS" + + "tatus.proto\"g\n\020MetaRegionServer\022\033\n\006serve" + + "r\030\001 \002(\0132\013.ServerName\022\023\n\013rpc_version\030\002 \001(" + + "\r\022!\n\005state\030\003 \001(\0162\022.RegionState.State\":\n\006" + + "Master\022\033\n\006master\030\001 \002(\0132\013.ServerName\022\023\n\013r" + + "pc_version\030\002 \001(\r\"\037\n\tClusterUp\022\022\n\nstart_d" + + "ate\030\001 \002(\t\"\210\001\n\020RegionTransition\022\027\n\017event_" + + "type_code\030\001 \002(\r\022\023\n\013region_name\030\002 \002(\014\022\023\n\013" + + "create_time\030\003 \002(\004\022 \n\013server_name\030\004 \002(\0132\013" + + ".ServerName\022\017\n\007payload\030\005 \001(\014\"\214\002\n\014SplitLo", + "gTask\022\"\n\005state\030\001 \002(\0162\023.SplitLogTask.Stat" + + "e\022 \n\013server_name\030\002 \002(\0132\013.ServerName\0221\n\004m" + + "ode\030\003 \001(\0162\032.SplitLogTask.RecoveryMode:\007U" + + "NKNOWN\"C\n\005State\022\016\n\nUNASSIGNED\020\000\022\t\n\005OWNED" + + "\020\001\022\014\n\010RESIGNED\020\002\022\010\n\004DONE\020\003\022\007\n\003ERR\020\004\">\n\014R" + + "ecoveryMode\022\013\n\007UNKNOWN\020\000\022\021\n\rLOG_SPLITTIN" + + "G\020\001\022\016\n\nLOG_REPLAY\020\002\"n\n\005Table\022$\n\005state\030\001 " + + "\002(\0162\014.Table.State:\007ENABLED\"?\n\005State\022\013\n\007E" + + "NABLED\020\000\022\014\n\010DISABLED\020\001\022\r\n\tDISABLING\020\002\022\014\n" + + "\010ENABLING\020\003\"%\n\017ReplicationPeer\022\022\n\ncluste", + "rkey\030\001 \002(\t\"^\n\020ReplicationState\022&\n\005state\030" + + "\001 \002(\0162\027.ReplicationState.State\"\"\n\005State\022" + + "\013\n\007ENABLED\020\000\022\014\n\010DISABLED\020\001\"+\n\027Replicatio" + + "nHLogPosition\022\020\n\010position\030\001 \002(\003\"%\n\017Repli" + + "cationLock\022\022\n\nlock_owner\030\001 \002(\t\"\230\001\n\tTable" + + "Lock\022\036\n\ntable_name\030\001 \001(\0132\n.TableName\022\037\n\n" + + "lock_owner\030\002 \001(\0132\013.ServerName\022\021\n\tthread_" + + "id\030\003 \001(\003\022\021\n\tis_shared\030\004 \001(\010\022\017\n\007purpose\030\005" + + " \001(\t\022\023\n\013create_time\030\006 \001(\003\";\n\017StoreSequen" + + "ceId\022\023\n\013family_name\030\001 \002(\014\022\023\n\013sequence_id", + "\030\002 \002(\004\"g\n\026RegionStoreSequenceIds\022 \n\030last" + + "_flushed_sequence_id\030\001 \002(\004\022+\n\021store_sequ" + + "ence_id\030\002 \003(\0132\020.StoreSequenceIdBE\n*org.a" + + "pache.hadoop.hbase.protobuf.generatedB\017Z" + + "ooKeeperProtosH\001\210\001\001\240\001\001" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { @@ -9626,7 +9758,7 @@ public final class ZooKeeperProtos { internal_static_MetaRegionServer_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_MetaRegionServer_descriptor, - new java.lang.String[] { "Server", "RpcVersion", }); + new java.lang.String[] { "Server", "RpcVersion", "State", }); internal_static_Master_descriptor = getDescriptor().getMessageTypes().get(1); internal_static_Master_fieldAccessorTable = new @@ -9706,6 +9838,7 @@ public final class ZooKeeperProtos { .internalBuildGeneratedFileFrom(descriptorData, new com.google.protobuf.Descriptors.FileDescriptor[] { org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.getDescriptor(), + org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos.getDescriptor(), }, assigner); } diff --git a/hbase-protocol/src/main/protobuf/ZooKeeper.proto b/hbase-protocol/src/main/protobuf/ZooKeeper.proto index 37816da..dc84798 100644 --- a/hbase-protocol/src/main/protobuf/ZooKeeper.proto +++ b/hbase-protocol/src/main/protobuf/ZooKeeper.proto @@ -26,6 +26,7 @@ option java_generate_equals_and_hash = true; option optimize_for = SPEED; import "HBase.proto"; +import "ClusterStatus.proto"; /** * Content of the meta-region-server znode. @@ -37,6 +38,8 @@ message MetaRegionServer { // clients connecting to the cluster can have prior knowledge of what version // to send to a RegionServer. AsyncHBase will use this to detect versions. optional uint32 rpc_version = 2; + // State of the region transition. OPEN means fully operational 'hbase:meta' + optional RegionState.State state = 3; } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java index c9ad041..601c23c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java @@ -768,7 +768,7 @@ public class AssignmentManager extends ZooKeeperListener { if (regionInfo.isMetaRegion()) { // If it's meta region, reset the meta location. // So that master knows the right meta region server. - MetaRegionTracker.setMetaLocation(watcher, sn); + MetaRegionTracker.setMetaLocation(watcher, sn, State.OPEN); } else { // No matter the previous server is online or offline, // we need to reset the last region server of the region. @@ -2937,31 +2937,35 @@ public class AssignmentManager extends ZooKeeperListener { processRegionInTransition(encodedRegionName, null); } } else if (!useZKForAssignment) { - // We need to send RPC call again for PENDING_OPEN/PENDING_CLOSE regions - // in case the RPC call is not sent out yet before the master was shut down - // since we update the state before we send the RPC call. We can't update - // the state after the RPC call. Otherwise, we don't know what's happened - // to the region if the master dies right after the RPC call is out. - Map rits = regionStates.getRegionsInTransition(); - for (RegionState regionState: rits.values()) { - if (!serverManager.isServerOnline(regionState.getServerName())) { - continue; // SSH will handle it - } - State state = regionState.getState(); - LOG.info("Processing " + regionState); - switch (state) { - case CLOSED: - invokeAssign(regionState.getRegion()); - break; - case PENDING_OPEN: - retrySendRegionOpen(regionState); - break; - case PENDING_CLOSE: - retrySendRegionClose(regionState); - break; - default: - // No process for other states - } + processRegionInTransitionZkLess(); + } + } + + void processRegionInTransitionZkLess() { + // We need to send RPC call again for PENDING_OPEN/PENDING_CLOSE regions + // in case the RPC call is not sent out yet before the master was shut down + // since we update the state before we send the RPC call. We can't update + // the state after the RPC call. Otherwise, we don't know what's happened + // to the region if the master dies right after the RPC call is out. + Map rits = regionStates.getRegionsInTransition(); + for (RegionState regionState : rits.values()) { + if (!serverManager.isServerOnline(regionState.getServerName())) { + continue; // SSH will handle it + } + State state = regionState.getState(); + LOG.info("Processing " + regionState); + switch (state) { + case CLOSED: + invokeAssign(regionState.getRegion()); + break; + case PENDING_OPEN: + retrySendRegionOpen(regionState); + break; + case PENDING_CLOSE: + retrySendRegionClose(regionState); + break; + default: + // No process for other states } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 0e591bb..c771b35 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -25,6 +25,7 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -219,6 +220,7 @@ import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils; import org.apache.hadoop.hbase.trace.SpanReceiverHost; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.CompressionTest; +import org.apache.hadoop.hbase.util.ConfigUtil; import org.apache.hadoop.hbase.util.FSTableDescriptors; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.HFileArchiveUtil; @@ -1001,16 +1003,22 @@ MasterServices, Server { status.setStatus("Assigning hbase:meta region"); RegionStates regionStates = assignmentManager.getRegionStates(); - regionStates.createRegionState(HRegionInfo.FIRST_META_REGIONINFO); - boolean rit = this.assignmentManager - .processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO); + + RegionState regionState = this.catalogTracker.getMetaRegionState(); + ServerName currentMetaServer = regionState.getServerName(); + regionStates.createRegionState(HRegionInfo.FIRST_META_REGIONINFO, + regionState.getState(), currentMetaServer); + boolean rit = + this.assignmentManager + .processRegionInTransitionAndBlockUntilAssigned(HRegionInfo.FIRST_META_REGIONINFO); boolean metaRegionLocation = this.catalogTracker.verifyMetaRegionLocation(timeout); - ServerName currentMetaServer = this.catalogTracker.getMetaLocation(); - if (!metaRegionLocation) { + if (!metaRegionLocation || !regionState.isOpened()) { // Meta location is not verified. It should be in transition, or offline. // We will wait for it to be assigned in enableSSHandWaitForMeta below. assigned++; - if (!rit) { + if (!ConfigUtil.useZKForAssignment(conf)) { + assignMetaZkLess(regionStates, regionState, timeout, previouslyFailedMetaRSs); + } else if (!rit) { // Assign meta since not already in transition if (currentMetaServer != null) { // If the meta server is not known to be dead or online, @@ -1057,6 +1065,24 @@ MasterServices, Server { status.setStatus("META assigned."); } + private void assignMetaZkLess(RegionStates regionStates, RegionState regionState, long timeout, + Set previouslyFailedRs) throws IOException, KeeperException { + ServerName currentServer = regionState.getServerName(); + if (serverManager.isServerOnline(currentServer)) { + LOG.info("Meta was in transition on " + currentServer); + assignmentManager.processRegionInTransitionZkLess(); + } else { + if (currentServer != null) { + splitMetaLogBeforeAssignment(currentServer); + regionStates.logSplit(HRegionInfo.FIRST_META_REGIONINFO); + previouslyFailedRs.add(currentServer); + } + LOG.info("Re-assigning hbase:meta, it was on " + currentServer); + regionStates.updateRegionState(HRegionInfo.FIRST_META_REGIONINFO, State.OFFLINE); + assignmentManager.assignMeta(); + } + } + void initNamespace() throws IOException { //create namespace manager tableNamespaceManager = new TableNamespaceManager(this); @@ -3223,7 +3249,9 @@ MasterServices, Server { RegionStateTransition rt = req.getTransition(0); TableName tableName = ProtobufUtil.toTableName( rt.getRegionInfo(0).getTableName()); - if (!TableName.META_TABLE_NAME.equals(tableName) + RegionStates regionStates = assignmentManager.getRegionStates(); + if (!(TableName.META_TABLE_NAME.equals(tableName) + && regionStates.getRegionState(HRegionInfo.FIRST_META_REGIONINFO) != null) && !assignmentManager.isFailoverCleanupDone()) { // Meta region is assigned before master finishes the // failover cleanup. So no need this check for it diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java index 1f2df14..45c4fd2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStateStore.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.catalog.CatalogTracker; import org.apache.hadoop.hbase.catalog.MetaEditor; +import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.master.RegionState.State; @@ -40,6 +41,8 @@ import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ConfigUtil; import org.apache.hadoop.hbase.util.MultiHConnection; +import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker; +import org.apache.zookeeper.KeeperException; import com.google.common.base.Preconditions; @@ -52,6 +55,7 @@ public class RegionStateStore { private static final Log LOG = LogFactory.getLog(RegionStateStore.class); private volatile HRegion metaRegion; + private volatile HRegion rootRegion; private volatile boolean initialized; private final boolean noPersistence; @@ -128,7 +132,6 @@ public class RegionStateStore { // increasing this value might improve the write throughput. multiHConnection = new MultiHConnection(conf, conf.getInt("hbase.regionstatestore.meta.connection", 1)); - } } initialized = true; @@ -141,22 +144,32 @@ public class RegionStateStore { } } - void updateRegionState(long openSeqNum, - RegionState newState, RegionState oldState) { - if (noPersistence || !initialized) { - return; - } - - HRegionInfo hri = newState.getRegion(); - if (!shouldPersistStateChange(hri, newState, oldState)) { + void updateRegionState(long openSeqNum, RegionState newState, RegionState oldState) { + if (noPersistence) { return; } + try { + HRegionInfo hri = newState.getRegion(); + // update meta before checking for initialization. + // meta state stored in zk. + if (hri.isMetaRegion()) { + // persist meta state in MetaRegionTracker + try { + MetaRegionTracker.setMetaLocation(server.getZooKeeper(), newState.getServerName(), + newState.getState()); + return; // Done + } catch (KeeperException e) { + throw new IOException("Failed to update meta ZNode", e); + } + } - ServerName oldServer = oldState != null ? oldState.getServerName() : null; - ServerName serverName = newState.getServerName(); - State state = newState.getState(); + if (!initialized || !shouldPersistStateChange(hri, newState, oldState)) { + return; + } - try { + ServerName oldServer = oldState != null ? oldState.getServerName() : null; + ServerName serverName = newState.getServerName(); + State state = newState.getState(); Put put = new Put(hri.getRegionName()); StringBuilder info = new StringBuilder("Updating row "); info.append(hri.getRegionNameAsString()).append(" with state=").append(state); @@ -180,8 +193,6 @@ public class RegionStateStore { put.addImmutable(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER, Bytes.toBytes(state.name())); LOG.info(info); - - // Persist the state change to meta if (metaRegion != null) { try { @@ -223,4 +234,5 @@ public class RegionStateStore { HRegionInfo a, HRegionInfo b, ServerName sn) throws IOException { MetaEditor.mergeRegions(catalogTracker, p, a, b, sn); } + } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index d39311e..6ce5a57 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -124,6 +124,7 @@ import org.apache.hadoop.hbase.ipc.RpcServer.BlockingServiceAndInterface; import org.apache.hadoop.hbase.ipc.RpcServerInterface; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; import org.apache.hadoop.hbase.ipc.ServerRpcController; +import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.master.SplitLogManager; import org.apache.hadoop.hbase.master.TableLockManager; import org.apache.hadoop.hbase.procedure.RegionServerProcedureManagerHost; @@ -1842,15 +1843,18 @@ public class HRegionServer implements ClientProtos.ClientService.BlockingInterfa // Update flushed sequence id of a recovering region in ZK updateRecoveringRegionLastFlushedSequenceId(r); - // Update ZK, or META - if (r.getRegionInfo().isMetaRegion()) { - MetaRegionTracker.setMetaLocation(getZooKeeper(), - this.serverNameFromMasterPOV); - } else if (useZKForAssignment) { - MetaEditor.updateRegionLocation(ct, r.getRegionInfo(), - this.serverNameFromMasterPOV, openSeqNum); + if (useZKForAssignment) { + if (r.getRegionInfo().isMetaRegion()) { + LOG.info("Updating zk with meta location"); + // We are setting State.OPEN as we need to set some state, but for zk assignment + // the value doesn't matter + MetaRegionTracker.setMetaLocation(getZooKeeper(), this.serverNameFromMasterPOV, State.OPEN); + } else { + MetaEditor.updateRegionLocation(ct, r.getRegionInfo(), this.serverNameFromMasterPOV, + openSeqNum); + } } - if (!useZKForAssignment + if (!useZKForAssignment && !reportRegionStateTransition(TransitionCode.OPENED, openSeqNum, r.getRegionInfo())) { throw new IOException("Failed to report opened region to master: " + r.getRegionNameAsString()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 9023e06..2673e5a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -3426,6 +3426,16 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { boolean failIfTimeout, Predicate predicate) throws E { return Waiter.waitFor(this.conf, timeout, interval, failIfTimeout, predicate); } + + /** + * Wait until no regions in transition. + * @param timeout How long to wait. + * @throws Exception + */ + public void waitUntilNoRegionsInTransition( + final long timeout) throws Exception { + waitFor(timeout, predicateNoRegionsInTransition()); + } /** * Returns a {@link Predicate} for checking that there are no regions in transition in master diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java index caeafbd..279062a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestCatalogTracker.java @@ -18,7 +18,10 @@ */ package org.apache.hadoop.hbase.catalog; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.IOException; import java.net.ConnectException; @@ -43,6 +46,8 @@ import org.apache.hadoop.hbase.client.HConnectionManager; import org.apache.hadoop.hbase.client.HConnectionTestingUtility; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; +import org.apache.hadoop.hbase.master.RegionState; +import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; @@ -137,7 +142,7 @@ public class TestCatalogTracker { constructAndStartCatalogTracker(connection); MetaRegionTracker.setMetaLocation(this.watcher, - ServerName.valueOf("example.com", 1234, System.currentTimeMillis())); + ServerName.valueOf("example.com", 1234, System.currentTimeMillis()), State.OPEN); } /** @@ -190,7 +195,7 @@ public class TestCatalogTracker { // Now start up the catalogtracker with our doctored Connection. final CatalogTracker ct = constructAndStartCatalogTracker(connection); - MetaRegionTracker.setMetaLocation(this.watcher, SN); + MetaRegionTracker.setMetaLocation(this.watcher, SN, State.OPEN); long timeout = UTIL.getConfiguration(). getLong("hbase.catalog.verification.timeout", 1000); Assert.assertFalse(ct.verifyMetaRegionLocation(timeout)); @@ -251,7 +256,7 @@ public class TestCatalogTracker { final CatalogTracker ct = constructAndStartCatalogTracker(connection); MetaRegionTracker.setMetaLocation(this.watcher, - ServerName.valueOf("example.com", 1234, System.currentTimeMillis())); + ServerName.valueOf("example.com", 1234, System.currentTimeMillis()), State.OPEN); Assert.assertFalse(ct.verifyMetaRegionLocation(100)); } @@ -286,9 +291,39 @@ public class TestCatalogTracker { // Now meta is available. Assert.assertTrue(ct.getMetaLocation().equals(hsa)); } - + + + /** + * Test normal operations + */ + @Test + public void testMetaLookup() + throws IOException, InterruptedException, ServiceException, KeeperException { + HConnection connection = Mockito.mock(HConnection.class); + final CatalogTracker ct = constructAndStartCatalogTracker(connection); + + assertNull(ct.getMetaLocation()); + for (RegionState.State state : RegionState.State.values()) { + if (state.equals(RegionState.State.OPEN)) + continue; + MetaRegionTracker.setMetaLocation(this.watcher, SN, state); + assertNull(ct.getMetaLocation()); + assertEquals(state, ct.getMetaRegionState().getState()); + } + MetaRegionTracker.setMetaLocation(this.watcher, SN, RegionState.State.OPEN); + assertEquals(ct.getMetaLocation(), SN); + assertEquals(RegionState.State.OPEN, + ct.getMetaRegionState().getState()); + + MetaRegionTracker.deleteMetaLocation(this.watcher); + assertNull(ct.getMetaRegionState().getServerName()); + assertEquals(ct.getMetaRegionState().getState(), + RegionState.State.OFFLINE); + assertNull(ct.getMetaLocation()); + } + private ServerName setMetaLocation() throws KeeperException { - MetaRegionTracker.setMetaLocation(this.watcher, SN); + MetaRegionTracker.setMetaLocation(this.watcher, SN, State.OPEN); return SN; } @@ -385,7 +420,7 @@ public class TestCatalogTracker { void doWaiting() throws InterruptedException { try { - while (this.ct.waitForMeta(100) == null); + while (this.ct.waitForMeta(10000) == null); } catch (NotAllMetaRegionsOnlineException e) { // Ignore. } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java index 76a440f..edf252d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java @@ -48,11 +48,13 @@ import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.RegionTransition; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.catalog.CatalogTracker; import org.apache.hadoop.hbase.catalog.MetaEditor; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.executor.EventType; import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.RequestConverter; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.RegionMergeTransaction; @@ -64,6 +66,7 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread; import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; import org.apache.hadoop.hbase.util.Threads; +import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker; import org.apache.hadoop.hbase.zookeeper.ZKAssign; import org.apache.hadoop.hbase.zookeeper.ZKTable; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; @@ -1256,5 +1259,149 @@ public class TestMasterFailover { // Done, shutdown the cluster TEST_UTIL.shutdownMiniCluster(); } + + /** + * Test meta in transition when master failover + */ + @Test(timeout = 180000) + public void testMetaInTransitionWhenMasterFailover() throws Exception { + final int NUM_MASTERS = 1; + final int NUM_RS = 1; + + // Start the cluster + Configuration conf = HBaseConfiguration.create(); + conf.setBoolean("hbase.assignment.usezk", false); + HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf); + TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS); + MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster(); + CatalogTracker catalogTracker = new CatalogTracker(HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL), + conf, null); + + log("Cluster started"); + + log("Moving meta off the master"); + HMaster activeMaster = cluster.getMaster(); + HRegionServer rs = cluster.getRegionServer(0); + ServerName metaServerName = cluster.getLiveRegionServerThreads() + .get(0).getRegionServer().getServerName(); + activeMaster.move(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), + Bytes.toBytes(metaServerName.getServerName())); + TEST_UTIL.waitUntilNoRegionsInTransition(60000); + assertEquals("Meta should be assigned on expected regionserver", + metaServerName, catalogTracker.getMetaLocation()); + + // Now kill master, root should remain on rs, where we placed it before. + log("Aborting master"); + activeMaster.stop("test-kill"); + cluster.waitForMasterToStop(activeMaster.getServerName(), 30000); + log("Master has aborted"); + + // r should remain where it was + RegionState metaState = + catalogTracker.getMetaRegionState(); + assertEquals("hbase:root should be onlined on RS", + metaState.getServerName(), rs.getServerName()); + assertEquals("hbase:root should be onlined on RS", + metaState.getState(), State.OPEN); + + // Start up a new master + log("Starting up a new master"); + activeMaster = cluster.startMaster().getMaster(); + log("Waiting for master to be ready"); + cluster.waitForActiveAndReadyMaster(); + log("Master is ready"); + + // ensure root is still deployed on RS + metaState = + catalogTracker.getMetaRegionState(); + assertEquals("hbase:root should be onlined on RS", + metaState.getServerName(), rs.getServerName()); + assertEquals("hbase:root should be onlined on RS", + metaState.getState(), State.OPEN); + + // Update root state as PENDING_OPEN, then kill master + // that simulates, that RS successfully deployed, but + // RPC was lost right before failure. + // region server should expire (how it can be verified?) + MetaRegionTracker.setMetaLocation(activeMaster.getZooKeeper(), + rs.getServerName(), State.PENDING_OPEN); + HRegion meta = rs.getFromOnlineRegions(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName()); + rs.removeFromOnlineRegions(meta, null); + meta.close(); + + log("Aborting master"); + activeMaster.stop("test-kill"); + cluster.waitForMasterToStop(activeMaster.getServerName(), 30000); + log("Master has aborted"); + + // Start up a new master + log("Starting up a new master"); + activeMaster = cluster.startMaster().getMaster(); + log("Waiting for master to be ready"); + cluster.waitForActiveAndReadyMaster(); + log("Master is ready"); + + TEST_UTIL.waitUntilNoRegionsInTransition(60000); + log("ROOT was assigned"); + + metaState = + catalogTracker.getMetaRegionState(); + assertEquals("hbase:root should be onlined on RS", + metaState.getServerName(), rs.getServerName()); + assertEquals("hbase:root should be onlined on RS", + metaState.getState(), State.OPEN); + + // Update root state as PENDING_CLOSE, then kill master + // that simulates, that RS successfully deployed, but + // RPC was lost right before failure. + // region server should expire (how it can be verified?) + MetaRegionTracker.setMetaLocation(activeMaster.getZooKeeper(), + rs.getServerName(), State.PENDING_CLOSE); + + log("Aborting master"); + activeMaster.stop("test-kill"); + cluster.waitForMasterToStop(activeMaster.getServerName(), 30000); + log("Master has aborted"); + + rs.closeRegion(null, RequestConverter.buildCloseRegionRequest( + rs.getServerName(), HRegionInfo.FIRST_META_REGIONINFO.getEncodedName(), false)); + + // Start up a new master + log("Starting up a new master"); + activeMaster = cluster.startMaster().getMaster(); + log("Waiting for master to be ready"); + cluster.waitForActiveAndReadyMaster(); + log("Master is ready"); + + TEST_UTIL.waitUntilNoRegionsInTransition(60000); + log("Meta was assigned"); + + rs.closeRegion(null, RequestConverter.buildCloseRegionRequest( + rs.getServerName(), HRegionInfo.FIRST_META_REGIONINFO.getEncodedName(), false)); + + // Set a dummy server to check if master reassigns meta on restart + MetaRegionTracker.setMetaLocation(activeMaster.getZooKeeper(), + ServerName.valueOf("dummyserver.example.org", 1234, -1L), State.OPEN); + + log("Aborting master"); + activeMaster.stop("test-kill"); + + cluster.waitForMasterToStop(activeMaster.getServerName(), 30000); + log("Master has aborted"); + + // Start up a new master + log("Starting up a new master"); + activeMaster = cluster.startMaster().getMaster(); + log("Waiting for master to be ready"); + cluster.waitForActiveAndReadyMaster(); + log("Master is ready"); + + TEST_UTIL.waitUntilNoRegionsInTransition(60000); + catalogTracker.verifyMetaRegionLocation(10000); + log("Meta was assigned"); + + // Done, shutdown the cluster + TEST_UTIL.shutdownMiniCluster(); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java index 50678da..7fdc09a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterNoCluster.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.catalog.MetaMockingUtil; import org.apache.hadoop.hbase.client.HConnection; import org.apache.hadoop.hbase.client.HConnectionTestingUtility; import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.regionserver.RegionOpeningState; import org.apache.hadoop.hbase.util.FSUtils; @@ -156,7 +157,7 @@ public class TestMasterNoCluster { final MockRegionServer rs2 = new MockRegionServer(conf, sn2); // Put some data into the servers. Make it look like sn0 has the metaH // Put data into sn2 so it looks like it has a few regions for a table named 't'. - MetaRegionTracker.setMetaLocation(rs0.getZooKeeper(), rs0.getServerName()); + MetaRegionTracker.setMetaLocation(rs0.getZooKeeper(), rs0.getServerName(), State.OPEN); final TableName tableName = TableName.valueOf("t"); Result [] results = new Result [] { MetaMockingUtil.getMetaTableRowResult( @@ -343,7 +344,7 @@ public class TestMasterNoCluster { // when its figured it just opened the meta region by setting the meta // location up into zk. Since we're mocking regionserver, need to do this // ourselves. - MetaRegionTracker.setMetaLocation(rs0.getZooKeeper(), rs0.getServerName()); + MetaRegionTracker.setMetaLocation(rs0.getZooKeeper(), rs0.getServerName(), State.OPEN); // Master should now come up. while (!master.isInitialized()) {Threads.sleep(10);} assertTrue(master.isInitialized());