diff --git a/src/main/java/org/apache/hadoop/hbase/HConstants.java b/src/main/java/org/apache/hadoop/hbase/HConstants.java index a44a0b9..615fbde 100644 --- a/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -364,6 +364,9 @@ public final class HConstants { public static final String HBASE_MASTER_LOGCLEANER_PLUGINS = "hbase.master.logcleaner.plugins"; + public static final String KEY_FOR_HOSTNAME_SEEN_BY_MASTER = + "hbase.regionserver.hostname.seen.by.master"; + private HConstants() { // Can't be instantiated with this ctor. } diff --git a/src/main/java/org/apache/hadoop/hbase/HMsg.java b/src/main/java/org/apache/hadoop/hbase/HMsg.java deleted file mode 100644 index c53460f..0000000 --- a/src/main/java/org/apache/hadoop/hbase/HMsg.java +++ /dev/null @@ -1,256 +0,0 @@ -/** - * Copyright 2010 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase; - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.io.Writable; - -/** - * HMsg is used to send messages between master and regionservers. Messages are - * sent as payload on the regionserver-to-master heartbeats. Region assignment - * does not use this mechanism. It goes via zookeeper. - * - *

Most of the time the messages are simple but some messages are accompanied - * by the region affected. HMsg may also carry an optional message. - * - *

TODO: Clean out all messages that go from master to regionserver; by - * design, these are to go via zk from here on out. - */ -public class HMsg implements Writable { - public static final HMsg [] STOP_REGIONSERVER_ARRAY = - new HMsg [] {new HMsg(Type.STOP_REGIONSERVER)}; - public static final HMsg [] EMPTY_HMSG_ARRAY = new HMsg[0]; - - public static enum Type { - /** Master tells region server to stop. - */ - STOP_REGIONSERVER, - - /** - * Region server split the region associated with this message. - */ - REGION_SPLIT, - - /** - * When RegionServer receives this message, it goes into a sleep that only - * an exit will cure. This message is sent by unit tests simulating - * pathological states. - */ - TESTING_BLOCK_REGIONSERVER, - } - - private Type type = null; - private HRegionInfo info = null; - private byte[] message = null; - private HRegionInfo daughterA = null; - private HRegionInfo daughterB = null; - - /** Default constructor. Used during deserialization */ - public HMsg() { - this(null); - } - - /** - * Construct a message with the specified message and empty HRegionInfo - * @param type Message type - */ - public HMsg(final HMsg.Type type) { - this(type, new HRegionInfo(), null); - } - - /** - * Construct a message with the specified message and HRegionInfo - * @param type Message type - * @param hri Region to which message type applies - */ - public HMsg(final HMsg.Type type, final HRegionInfo hri) { - this(type, hri, null); - } - - /** - * Construct a message with the specified message and HRegionInfo - * - * @param type Message type - * @param hri Region to which message type applies. Cannot be - * null. If no info associated, used other Constructor. - * @param msg Optional message (Stringified exception, etc.) - */ - public HMsg(final HMsg.Type type, final HRegionInfo hri, final byte[] msg) { - this(type, hri, null, null, msg); - } - - /** - * Construct a message with the specified message and HRegionInfo - * - * @param type Message type - * @param hri Region to which message type applies. Cannot be - * null. If no info associated, used other Constructor. - * @param daughterA - * @param daughterB - * @param msg Optional message (Stringified exception, etc.) - */ - public HMsg(final HMsg.Type type, final HRegionInfo hri, - final HRegionInfo daughterA, final HRegionInfo daughterB, final byte[] msg) { - this.type = type; - if (hri == null) { - throw new NullPointerException("Region cannot be null"); - } - this.info = hri; - this.message = msg; - this.daughterA = daughterA; - this.daughterB = daughterB; - } - - /** - * @return Region info or null if none associated with this message type. - */ - public HRegionInfo getRegionInfo() { - return this.info; - } - - /** @return the type of message */ - public Type getType() { - return this.type; - } - - /** - * @param other Message type to compare to - * @return True if we are of same message type as other - */ - public boolean isType(final HMsg.Type other) { - return this.type.equals(other); - } - - /** @return the message type */ - public byte[] getMessage() { - return this.message; - } - - /** - * @return First daughter if Type is MSG_REPORT_SPLIT_INCLUDES_DAUGHTERS else - * null - */ - public HRegionInfo getDaughterA() { - return this.daughterA; - } - - /** - * @return Second daughter if Type is MSG_REPORT_SPLIT_INCLUDES_DAUGHTERS else - * null - */ - public HRegionInfo getDaughterB() { - return this.daughterB; - } - - /** - * @see java.lang.Object#toString() - */ - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(this.type.toString()); - // If null or empty region, don't bother printing it out. - if (this.info != null && this.info.getRegionName().length > 0) { - sb.append(": "); - sb.append(this.info.getRegionNameAsString()); - } - if (this.message != null && this.message.length > 0) { - sb.append(": " + Bytes.toString(this.message)); - } - return sb.toString(); - } - - /** - * @see java.lang.Object#equals(java.lang.Object) - */ - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - HMsg that = (HMsg)obj; - return this.type.equals(that.type) && - (this.info != null)? this.info.equals(that.info): - that.info == null; - } - - /** - * @see java.lang.Object#hashCode() - */ - @Override - public int hashCode() { - int result = this.type.hashCode(); - if (this.info != null) { - result ^= this.info.hashCode(); - } - return result; - } - - // //////////////////////////////////////////////////////////////////////////// - // Writable - ////////////////////////////////////////////////////////////////////////////// - - /** - * @see org.apache.hadoop.io.Writable#write(java.io.DataOutput) - */ - public void write(DataOutput out) throws IOException { - out.writeInt(this.type.ordinal()); - this.info.write(out); - if (this.message == null || this.message.length == 0) { - out.writeBoolean(false); - } else { - out.writeBoolean(true); - Bytes.writeByteArray(out, this.message); - } - if (this.type.equals(Type.REGION_SPLIT)) { - this.daughterA.write(out); - this.daughterB.write(out); - } - } - - /** - * @see org.apache.hadoop.io.Writable#readFields(java.io.DataInput) - */ - public void readFields(DataInput in) throws IOException { - int ordinal = in.readInt(); - this.type = HMsg.Type.values()[ordinal]; - this.info.readFields(in); - boolean hasMessage = in.readBoolean(); - if (hasMessage) { - this.message = Bytes.readByteArray(in); - } - if (this.type.equals(Type.REGION_SPLIT)) { - this.daughterA = new HRegionInfo(); - this.daughterB = new HRegionInfo(); - this.daughterA.readFields(in); - this.daughterB.readFields(in); - } - } -} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java b/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java index bd353b8..188dd97 100644 --- a/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java +++ b/src/main/java/org/apache/hadoop/hbase/HRegionLocation.java @@ -20,19 +20,17 @@ package org.apache.hadoop.hbase; /** - * Contains the HRegionInfo for the region and the HServerAddress for the - * HRegionServer serving the region + * Data structure to hold HRegionInfo and the HServerAddress for the hosting + * HRegionServer. Immutable. */ public class HRegionLocation implements Comparable { - // TODO: Is this class necessary? Why not just have a Pair? - private HRegionInfo regionInfo; - private HServerAddress serverAddress; + private final HRegionInfo regionInfo; + private final HServerAddress serverAddress; /** * Constructor - * * @param regionInfo the HRegionInfo for the region - * @param serverAddress the HServerAddress for the region server + * @param serverAddress the HServerAddress for the hosting region server */ public HRegionLocation(HRegionInfo regionInfo, HServerAddress serverAddress) { this.regionInfo = regionInfo; @@ -44,8 +42,8 @@ public class HRegionLocation implements Comparable { */ @Override public String toString() { - return "address: " + this.serverAddress.toString() + ", regioninfo: " + - this.regionInfo.getRegionNameAsString(); + return "region=" + this.regionInfo.getRegionNameAsString() + + ", address=" + this.serverAddress.toString(); } /** @@ -91,7 +89,7 @@ public class HRegionLocation implements Comparable { public int compareTo(HRegionLocation o) { int result = this.regionInfo.compareTo(o.regionInfo); - if(result == 0) { + if (result == 0) { result = this.serverAddress.compareTo(o.serverAddress); } return result; diff --git a/src/main/java/org/apache/hadoop/hbase/HServerAddress.java b/src/main/java/org/apache/hadoop/hbase/HServerAddress.java index 7f8a472..ef879ff 100644 --- a/src/main/java/org/apache/hadoop/hbase/HServerAddress.java +++ b/src/main/java/org/apache/hadoop/hbase/HServerAddress.java @@ -19,25 +19,31 @@ */ package org.apache.hadoop.hbase; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.io.WritableComparable; - import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.net.InetSocketAddress; import java.net.InetAddress; +import java.net.InetSocketAddress; + +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.io.WritableComparable; /** - * HServerAddress is a "label" for a HBase server made of host and port number. + * HServerAddress hosts a {@link InetSocketAddress} making it + * {@link WritableComparable}. Resolves on construction AND on + * deserialization so could end up with different results if the + * two ends of serialization have different resolvers. + * Be careful where you use it. */ public class HServerAddress implements WritableComparable { - private InetSocketAddress address; - String stringValue; + private InetSocketAddress address = null; + private String cachedToString = ""; + /** + * Constructor for deserializing use only. + */ public HServerAddress() { - this.address = null; - this.stringValue = null; + super(); } /** @@ -46,34 +52,38 @@ public class HServerAddress implements WritableComparable { */ public HServerAddress(InetSocketAddress address) { this.address = address; - this.stringValue = address.getAddress().getHostName() + ":" + - address.getPort(); checkBindAddressCanBeResolved(); + this.cachedToString = createCachedToString(); } /** * @param hostAndPort Hostname and port formatted as <hostname> ':' <port> */ - public HServerAddress(String hostAndPort) { + public HServerAddress(final String hostAndPort) { + this(createInetSocketAddressFromHostAndPortStr(hostAndPort)); + } + + private static InetSocketAddress createInetSocketAddressFromHostAndPortStr( + final String hostAndPort) { int colonIndex = hostAndPort.lastIndexOf(':'); if (colonIndex < 0) { throw new IllegalArgumentException("Not a host:port pair: " + hostAndPort); } String host = hostAndPort.substring(0, colonIndex); int port = Integer.parseInt(hostAndPort.substring(colonIndex + 1)); - this.address = new InetSocketAddress(host, port); - this.stringValue = address.getHostName() + ":" + port; - checkBindAddressCanBeResolved(); + return new InetSocketAddress(host, port); + } + + private String createCachedToString() { + return this.address.toString(); } /** - * @param bindAddress Hostname + * @param hostname Hostname * @param port Port number */ - public HServerAddress(String bindAddress, int port) { - this.address = new InetSocketAddress(bindAddress, port); - this.stringValue = address.getHostName() + ":" + port; - checkBindAddressCanBeResolved(); + public HServerAddress(final String hostname, final int port) { + this(new InetSocketAddress(hostname, port)); } /** @@ -81,45 +91,48 @@ public class HServerAddress implements WritableComparable { * @param other HServerAddress to copy from */ public HServerAddress(HServerAddress other) { - String bindAddress = other.getBindAddress(); - int port = other.getPort(); - this.address = new InetSocketAddress(bindAddress, port); - stringValue = other.stringValue; - checkBindAddressCanBeResolved(); + this(new InetSocketAddress(other.getHostname(), other.getPort())); } - /** @return Bind address */ + /** @return Bind address -- the raw IP, the result of a call to + * {@link InetSocketAddress#getAddress()#getHostAddress()} -- + * or null if cannot resolve */ public String getBindAddress() { - final InetAddress addr = address.getAddress(); - if (addr != null) { - return addr.getHostAddress(); - } else { - LogFactory.getLog(HServerAddress.class).error("Could not resolve the" - + " DNS name of " + stringValue); - return null; - } + // This returns null if the address is not resolved. + final InetAddress addr = this.address.getAddress(); + if (addr != null) return addr.getHostAddress(); + LogFactory.getLog(HServerAddress.class).error("Could not resolve the" + + " DNS name of " + this.address.toString()); + return null; } private void checkBindAddressCanBeResolved() { if (getBindAddress() == null) { throw new IllegalArgumentException("Could not resolve the" - + " DNS name of " + stringValue); + + " DNS name of " + this.address.toString()); } } /** @return Port number */ public int getPort() { - return address.getPort(); + return this.address.getPort(); } /** @return Hostname */ public String getHostname() { - return address.getHostName(); + return this.address.getHostName(); + } + + /** + * @return Returns ':' + */ + public String getHostnameAndPort() { + return getHostname() + ":" + getPort(); } /** @return The InetSocketAddress */ public InetSocketAddress getInetSocketAddress() { - return address; + return this.address; } /** @@ -127,27 +140,21 @@ public class HServerAddress implements WritableComparable { */ @Override public String toString() { - return stringValue == null ? "" : stringValue; + return this.cachedToString; } @Override public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null) { - return false; - } - if (getClass() != o.getClass()) { - return false; - } - return compareTo((HServerAddress) o) == 0; + if (this == o) return true; + if (o == null) return false; + if (getClass() != o.getClass()) return false; + return compareTo((HServerAddress)o) == 0; } @Override public int hashCode() { - int result = address.hashCode(); - result ^= stringValue.hashCode(); + int result = this.address.hashCode(); + result ^= this.cachedToString.hashCode(); return result; } @@ -158,24 +165,20 @@ public class HServerAddress implements WritableComparable { public void readFields(DataInput in) throws IOException { String hostname = in.readUTF(); int port = in.readInt(); - - if (hostname == null || hostname.length() == 0) { - address = null; - stringValue = null; - } else { - address = new InetSocketAddress(hostname, port); - stringValue = hostname + ":" + port; + if (hostname != null && hostname.length() > 0) { + this.address = new InetSocketAddress(hostname, port); checkBindAddressCanBeResolved(); + createCachedToString(); } } public void write(DataOutput out) throws IOException { - if (address == null) { + if (this.address == null) { out.writeUTF(""); out.writeInt(0); } else { - out.writeUTF(address.getAddress().getHostName()); - out.writeInt(address.getPort()); + out.writeUTF(this.address.getAddress().getHostName()); + out.writeInt(this.address.getPort()); } } @@ -187,7 +190,7 @@ public class HServerAddress implements WritableComparable { // Addresses as Strings may not compare though address is for the one // server with only difference being that one address has hostname // resolved whereas other only has IP. - if (address.equals(o.address)) return 0; + if (this.address.equals(o.address)) return 0; return toString().compareTo(o.toString()); } -} +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/HServerInfo.java b/src/main/java/org/apache/hadoop/hbase/HServerInfo.java index c742951..82e70b3 100644 --- a/src/main/java/org/apache/hadoop/hbase/HServerInfo.java +++ b/src/main/java/org/apache/hadoop/hbase/HServerInfo.java @@ -22,69 +22,41 @@ package org.apache.hadoop.hbase; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Comparator; -import java.util.Set; import org.apache.hadoop.hbase.regionserver.HRegionServer; -import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; /** - * HServerInfo is meta info about an {@link HRegionServer}. It is the token - * by which a master distingushes a particular regionserver from the rest. - * It holds hostname, ports, regionserver startcode, and load. Each server has - * a servername where servername is made up of a concatenation of - * hostname, port, and regionserver startcode. This servername is used in - * various places identifying this regionserver. Its even used as part of - * a pathname in the filesystem. As part of the initialization, - * master will pass the regionserver the address that it knows this regionserver - * by. In subsequent communications, the regionserver will pass a HServerInfo - * with the master-supplied address. + * HServerInfo is meta info about an {@link HRegionServer}. It hosts the + * {@link HServerAddress}, its webui port, and its server startcode. */ public class HServerInfo implements WritableComparable { - /* - * This character is used as separator between server hostname and port and - * its startcode. Servername is formatted as - * <hostname> '{@ink #SERVERNAME_SEPARATOR"}' <port> '{@ink #SERVERNAME_SEPARATOR"}' <startcode>. - */ - private static final String SERVERNAME_SEPARATOR = ","; - + // You used to get the 'serverName' from this class but now you must go to + // ServerName instances. private HServerAddress serverAddress; private long startCode; - private HServerLoad load; - private int infoPort; - // Servername is made of hostname, port and startcode. - private String serverName = null; - // Hostname of the regionserver. - private String hostname; - private String cachedHostnamePort = null; + private int webuiport; + private String cachedToString = null; public HServerInfo() { - this(new HServerAddress(), 0, HConstants.DEFAULT_REGIONSERVER_INFOPORT, - "default name"); + this(new HServerAddress(), 0, HConstants.DEFAULT_REGIONSERVER_INFOPORT); } /** - * Constructor that creates a HServerInfo with a generated startcode and an - * empty load. - * @param serverAddress An {@link InetSocketAddress} encased in a {@link Writable} - * @param infoPort Port the webui runs on. - * @param hostname Server hostname. + * Constructor that creates a HServerInfo with a generated startcode + * @param serverAddress + * @param webuiport Port the webui runs on. */ - public HServerInfo(HServerAddress serverAddress, final int infoPort, - final String hostname) { - this(serverAddress, System.currentTimeMillis(), infoPort, hostname); + public HServerInfo(final HServerAddress serverAddress, final int webuiport) { + this(serverAddress, System.currentTimeMillis(), webuiport); } public HServerInfo(HServerAddress serverAddress, long startCode, - final int infoPort, String hostname) { + final int webuiport) { this.serverAddress = serverAddress; this.startCode = startCode; - this.load = new HServerLoad(); - this.infoPort = infoPort; - this.hostname = hostname; + this.webuiport = webuiport; } /** @@ -94,106 +66,27 @@ public class HServerInfo implements WritableComparable { public HServerInfo(HServerInfo other) { this.serverAddress = new HServerAddress(other.getServerAddress()); this.startCode = other.getStartCode(); - this.load = other.getLoad(); - this.infoPort = other.getInfoPort(); - this.hostname = other.hostname; - } - - public HServerLoad getLoad() { - return load; - } - - public void setLoad(HServerLoad load) { - this.load = load; + this.webuiport = other.getInfoPort(); } public synchronized HServerAddress getServerAddress() { return new HServerAddress(serverAddress); } - public synchronized void setServerAddress(HServerAddress serverAddress) { - this.serverAddress = serverAddress; - this.hostname = serverAddress.getHostname(); - this.serverName = null; - } - public synchronized long getStartCode() { return startCode; } public int getInfoPort() { - return this.infoPort; - } - - public String getHostname() { - return this.hostname; - } - - /** - * @return The hostname and port concatenated with a ':' as separator. - */ - public synchronized String getHostnamePort() { - if (this.cachedHostnamePort == null) { - this.cachedHostnamePort = getHostnamePort(this.hostname, this.serverAddress.getPort()); - } - return this.cachedHostnamePort; - } - - /** - * @param hostname - * @param port - * @return The hostname and port concatenated with a ':' as separator. - */ - public static String getHostnamePort(final String hostname, final int port) { - return hostname + ":" + port; - } - - /** - * Gets the unique server instance name. Includes the hostname, port, and - * start code. - * @return Server name made of the concatenation of hostname, port and - * startcode formatted as <hostname> ',' <port> ',' <startcode> - */ - public synchronized String getServerName() { - if (this.serverName == null) { - this.serverName = getServerName(this.hostname, - this.serverAddress.getPort(), this.startCode); - } - return this.serverName; + return getWebuiPort(); } - public static synchronized String getServerName(final String hostAndPort, - final long startcode) { - int index = hostAndPort.indexOf(":"); - if (index <= 0) throw new IllegalArgumentException("Expected ':' "); - return getServerName(hostAndPort.substring(0, index), - Integer.parseInt(hostAndPort.substring(index + 1)), startcode); - } - - /** - * @param address Server address - * @param startCode Server startcode - * @return Server name made of the concatenation of hostname, port and - * startcode formatted as <hostname> ',' <port> ',' <startcode> - */ - public static String getServerName(HServerAddress address, long startCode) { - return getServerName(address.getHostname(), address.getPort(), startCode); + public int getWebuiPort() { + return this.webuiport; } - /* - * @param hostName - * @param port - * @param startCode - * @return Server name made of the concatenation of hostname, port and - * startcode formatted as <hostname> ',' <port> ',' <startcode> - */ - public static String getServerName(String hostName, int port, long startCode) { - StringBuilder name = new StringBuilder(hostName); - name.append(SERVERNAME_SEPARATOR); - name.append(port); - name.append(SERVERNAME_SEPARATOR); - name.append(startCode); - return name.toString(); + public String getHostname() { + return this.serverAddress.getHostname(); } /** @@ -202,81 +95,49 @@ public class HServerInfo implements WritableComparable { * @see #getLoad() */ @Override - public String toString() { - return "serverName=" + getServerName() + - ", load=(" + this.load.toString() + ")"; + public synchronized String toString() { + if (this.cachedToString == null) { + this.cachedToString = "servername=" + + ServerName.getServerName(this.serverAddress.getHostname(), + this.serverAddress.getPort(), this.startCode).toString() + + ", webuiport=" + this.webuiport; + } + return this.cachedToString; } @Override public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; return compareTo((HServerInfo)obj) == 0; } @Override public int hashCode() { - return this.getServerName().hashCode(); + int code = this.serverAddress.hashCode(); + code ^= this.webuiport; + code ^= this.startCode; + return code; } public void readFields(DataInput in) throws IOException { this.serverAddress.readFields(in); this.startCode = in.readLong(); - this.load.readFields(in); - this.infoPort = in.readInt(); - this.hostname = in.readUTF(); + this.webuiport = in.readInt(); } public void write(DataOutput out) throws IOException { this.serverAddress.write(out); out.writeLong(this.startCode); - this.load.write(out); - out.writeInt(this.infoPort); - out.writeUTF(hostname); + out.writeInt(this.webuiport); } public int compareTo(HServerInfo o) { - return this.getServerName().compareTo(o.getServerName()); - } - - /** - * Orders HServerInfos by load then name. Natural/ascending order. - */ - public static class LoadComparator implements Comparator { - @Override - public int compare(HServerInfo left, HServerInfo right) { - int loadCompare = left.getLoad().compareTo(right.getLoad()); - return loadCompare != 0 ? loadCompare : left.compareTo(right); - } - } - - /** - * Utility method that does a find of a servername or a hostandport combination - * in the passed Set. - * @param servers Set of server names - * @param serverName Name to look for - * @param hostAndPortOnly If serverName is a - * hostname ':' port - * or hostname , port , startcode. - * @return True if serverName found in servers - */ - public static boolean isServer(final Set servers, - final String serverName, final boolean hostAndPortOnly) { - if (!hostAndPortOnly) return servers.contains(serverName); - String serverNameColonReplaced = - serverName.replaceFirst(":", SERVERNAME_SEPARATOR); - for (String hostPortStartCode: servers) { - int index = hostPortStartCode.lastIndexOf(SERVERNAME_SEPARATOR); - String hostPortStrippedOfStartCode = hostPortStartCode.substring(0, index); - if (hostPortStrippedOfStartCode.equals(serverNameColonReplaced)) return true; - } - return false; + int compare = this.serverAddress.compareTo(o.getServerAddress()); + if (compare != 0) return compare; + if (this.webuiport != o.getInfoPort()) return this.webuiport - o.getInfoPort(); + if (this.startCode != o.getStartCode()) return (int)(this.startCode - o.getStartCode()); + return 0; } -} +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/HServerLoad.java b/src/main/java/org/apache/hadoop/hbase/HServerLoad.java index efa7e0e..4ebe9ea 100644 --- a/src/main/java/org/apache/hadoop/hbase/HServerLoad.java +++ b/src/main/java/org/apache/hadoop/hbase/HServerLoad.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.List; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Strings; @@ -32,7 +33,8 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; /** - * This class encapsulates metrics for determining the load on a HRegionServer + * RegionServer load. + * Immutable. */ public class HServerLoad implements WritableComparable { /** number of regions */ @@ -143,44 +145,6 @@ public class HServerLoad implements WritableComparable { return storefileIndexSizeMB; } - // Setters - - /** - * @param name the region name - */ - public void setName(byte[] name) { - this.name = name; - } - - /** - * @param stores the number of stores - */ - public void setStores(int stores) { - this.stores = stores; - } - - /** - * @param storefiles the number of storefiles - */ - public void setStorefiles(int storefiles) { - this.storefiles = storefiles; - } - - /** - * @param memstoreSizeMB the memstore size, in MB - */ - public void setMemStoreSizeMB(int memstoreSizeMB) { - this.memstoreSizeMB = memstoreSizeMB; - } - - /** - * @param storefileIndexSizeMB the approximate size of storefile indexes - * on the heap, in MB - */ - public void setStorefileIndexSizeMB(int storefileIndexSizeMB) { - this.storefileIndexSizeMB = storefileIndexSizeMB; - } - // Writable public void readFields(DataInput in) throws IOException { int namelen = in.readInt(); @@ -248,9 +212,22 @@ public class HServerLoad implements WritableComparable { */ public HServerLoad(final int numberOfRequests, final int usedHeapMB, final int maxHeapMB) { + this(numberOfRequests, usedHeapMB, maxHeapMB, null); + } + + /** + * Constructor + * @param numberOfRequests + * @param usedHeapMB + * @param maxHeapMB + * @param regionLoad + */ + public HServerLoad(final int numberOfRequests, final int usedHeapMB, + final int maxHeapMB, final List regionLoad) { this.numberOfRequests = numberOfRequests; this.usedHeapMB = usedHeapMB; this.maxHeapMB = maxHeapMB; + if (regionLoad != null) this.regionLoad.addAll(regionLoad); } /** @@ -373,8 +350,7 @@ public class HServerLoad implements WritableComparable { */ public int getStorefiles() { int count = 0; - for (RegionLoad info: regionLoad) - count += info.getStorefiles(); + for (RegionLoad info: regionLoad) count += info.getStorefiles(); return count; } @@ -408,60 +384,6 @@ public class HServerLoad implements WritableComparable { return count; } - // Setters - - /** - * @param numberOfRegions the number of regions - */ - public void setNumberOfRegions(int numberOfRegions) { - this.numberOfRegions = numberOfRegions; - } - - /** - * @param numberOfRequests the number of requests to set - */ - public void setNumberOfRequests(int numberOfRequests) { - this.numberOfRequests = numberOfRequests; - } - - /** - * @param usedHeapMB the amount of heap in use, in MB - */ - public void setUsedHeapMB(int usedHeapMB) { - this.usedHeapMB = usedHeapMB; - } - - /** - * @param maxHeapMB the maximum allowable heap size, in MB - */ - public void setMaxHeapMB(int maxHeapMB) { - this.maxHeapMB = maxHeapMB; - } - - /** - * @param load Instance of HServerLoad - */ - public void addRegionInfo(final HServerLoad.RegionLoad load) { - this.numberOfRegions++; - this.regionLoad.add(load); - } - - /** - * @param name - * @param stores - * @param storefiles - * @param memstoreSizeMB - * @param storefileIndexSizeMB - * @deprecated Use {@link #addRegionInfo(RegionLoad)} - */ - @Deprecated - public void addRegionInfo(final byte[] name, final int stores, - final int storefiles, final int storefileSizeMB, - final int memstoreSizeMB, final int storefileIndexSizeMB) { - this.regionLoad.add(new HServerLoad.RegionLoad(name, stores, storefiles, - storefileSizeMB, memstoreSizeMB, storefileIndexSizeMB)); - } - // Writable public void readFields(DataInput in) throws IOException { @@ -490,4 +412,4 @@ public class HServerLoad implements WritableComparable { public int compareTo(HServerLoad o) { return this.getLoad() - o.getLoad(); } -} +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/ServerName.java b/src/main/java/org/apache/hadoop/hbase/ServerName.java new file mode 100644 index 0000000..171abf0 --- /dev/null +++ b/src/main/java/org/apache/hadoop/hbase/ServerName.java @@ -0,0 +1,133 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.util.Set; + +/** + * Utility handling server names where server names are formatted + * <hostname> '{@link #SERVERNAME_SEPARATOR"}' <port> '{@ink #SERVERNAME_SEPARATOR"}' <startcode>. + * A server name is used uniquely identifying a server instance. The format is + * safe to use in the filesystem and as znode name up in ZooKeeper. + */ +public class ServerName { + /* + * This character is used as separator between server hostname and port and + * its startcode. + */ + static final String SERVERNAME_SEPARATOR = ","; + + private final String servername; + private final String hostname; + private final int port; + private final long startcode; + + public ServerName(final String hostname, final int port, final long startcode) { + this.hostname = hostname; + this.port = port; + this.startcode = startcode; + this.servername = getServerName(hostname, port, startcode); + } + + @Override + public String toString() { + return this.servername; + } + + public String getServername() { + return servername; + } + + public String getHostname() { + return hostname; + } + + public int getPort() { + return port; + } + + public long getStartcode() { + return startcode; + } + + /** + * @param hostName + * @param port + * @param startcode + * @return Server name made of the concatenation of hostname, port and + * startcode formatted as <hostname> ',' <port> ',' <startcode> + */ + public static String getServerName(String hostName, int port, long startcode) { + StringBuilder name = new StringBuilder(hostName); + name.append(SERVERNAME_SEPARATOR); + name.append(port); + name.append(SERVERNAME_SEPARATOR); + name.append(startcode); + return name.toString(); + } + + /** + * @param hostAndPort String in form of <hostname> ':' <port> + * @param startcode + * @return Server name made of the concatenation of hostname, port and + * startcode formatted as <hostname> ',' <port> ',' <startcode> + */ + public static synchronized String getServerName(final String hostAndPort, + final long startcode) { + int index = hostAndPort.indexOf(":"); + if (index <= 0) throw new IllegalArgumentException("Expected ':' "); + return getServerName(hostAndPort.substring(0, index), + Integer.parseInt(hostAndPort.substring(index + 1)), startcode); + } + + /** + * @param serverName ServerName in form specified by {@link #getServerName()} + * @return The server start code parsed from servername + */ + public static long getServerStartcodeFromServerName(final String serverName) { + int index = serverName.lastIndexOf(SERVERNAME_SEPARATOR); + return Long.parseLong(serverName.substring(index + 1)); + } + + /** + * Utility method that does a find of a servername or a hostandport combination + * in the passed Set. + * @param servers Set of server names where server name is formatted as + * a {@link ServerName}. + * @param serverName Name to look for formatted as a {@link ServerName} or + * as a hostname ':' port. + * @param hostAndPortOnly True if passed serverName is a + * hostname ':' port, false if its formatted as a + * {@link ServerName}. + * @return True if serverName found in servers + */ + public static boolean isServer(final Set servers, + final String serverName, final boolean hostAndPortOnly) { + if (!hostAndPortOnly) return servers.contains(serverName); + String serverNameColonReplaced = + serverName.replaceFirst(":", SERVERNAME_SEPARATOR); + for (String hostPortStartCode: servers) { + int index = hostPortStartCode.lastIndexOf(SERVERNAME_SEPARATOR); + String hostPortStrippedOfStartCode = hostPortStartCode.substring(0, index); + if (hostPortStrippedOfStartCode.equals(serverNameColonReplaced)) return true; + } + return false; + } +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java index efa677b..56c7e4b 100644 --- a/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java +++ b/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java @@ -405,8 +405,7 @@ public class MetaReader { final long startCode = Bytes.toLong(data.getValue(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER)); HServerAddress server = new HServerAddress(Bytes.toString(value)); - HServerInfo hsi = new HServerInfo(server, startCode, 0, - server.getHostname()); + HServerInfo hsi = new HServerInfo(server, startCode, 0); return new Pair(info, hsi); } else { return new Pair(info, null); @@ -578,7 +577,7 @@ public class MetaReader { * @throws IOException */ public static NavigableMap - getServerUserRegions(CatalogTracker catalogTracker, final HServerInfo hsi) + getServerUserRegions(CatalogTracker catalogTracker, final String serverName) throws IOException { HRegionInterface metaServer = catalogTracker.waitForMetaServerConnectionDefault(); @@ -594,7 +593,8 @@ public class MetaReader { Pair pair = metaRowToRegionPairWithInfo(result); if (pair == null) continue; - if (pair.getSecond() == null || !pair.getSecond().equals(hsi)) { + if (pair.getSecond() == null || + !pair.getSecond().getServerName().equals(serverName)) { continue; } hris.put(pair.getFirst(), result); diff --git a/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java b/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java index b48b390..27f3e55 100644 --- a/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java +++ b/src/main/java/org/apache/hadoop/hbase/executor/EventHandler.java @@ -221,4 +221,4 @@ public abstract class EventHandler implements Runnable, Comparable { public synchronized void setListener(EventHandlerListener listener) { this.listener = listener; } -} +} \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java b/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java index 7b8f193..7f3c9b0 100644 --- a/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java +++ b/src/main/java/org/apache/hadoop/hbase/io/HbaseObjectWritable.java @@ -41,7 +41,6 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HMsg; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; import org.apache.hadoop.hbase.HServerInfo; @@ -146,8 +145,14 @@ public class HbaseObjectWritable implements Writable, WritableWithSize, Configur // Hbase types addToMap(HColumnDescriptor.class, code++); addToMap(HConstants.Modify.class, code++); - addToMap(HMsg.class, code++); - addToMap(HMsg[].class, code++); + + // We used to have a class named HMsg but its been removed. Rather than + // just axe it, use following random Integer class -- we just chose any + // class from java.lang -- instead just so codes that follow stay + // in same relative place. + addToMap(Integer.class, code++); + addToMap(Integer[].class, code++); + addToMap(HRegion.class, code++); addToMap(HRegion[].class, code++); addToMap(HRegionInfo.class, code++); diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java index 25139b3..0209dea 100644 --- a/src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java +++ b/src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java @@ -19,22 +19,14 @@ */ package org.apache.hadoop.hbase.ipc; -import org.apache.hadoop.hbase.HMsg; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HServerInfo; +import java.io.IOException; + import org.apache.hadoop.io.MapWritable; import org.apache.hadoop.ipc.VersionedProtocol; -import java.io.IOException; - /** - * HRegionServers interact with the HMasterRegionInterface to report on local - * goings-on and to obtain data-handling instructions from the HMaster. - *

Changes here need to be reflected in HbaseObjectWritable HbaseRPC#Invoker. - * - *

NOTE: if you change the interface, you must change the RPC version - * number in HBaseRPCProtocolVersion - * + * The Master publishes this Interface for RegionServers to register themselves + * on. */ public interface HMasterRegionInterface extends VersionedProtocol { /** @@ -44,32 +36,18 @@ public interface HMasterRegionInterface extends VersionedProtocol { // maintained a single global version number on all HBase Interfaces. This // meant all HBase RPC was broke though only one of the three RPC Interfaces // had changed. This has since been undone. - public static final long VERSION = 28L; + public static final long VERSION = 29L; /** * Called when a region server first starts - * @param info server info + * @param port Port number this regionserver is up on. + * @param serverStartcode This servers startcode. * @param serverCurrentTime The current time of the region server in ms * @throws IOException e * @return Configuration for the regionserver to use: e.g. filesystem, * hbase rootdir, etc. */ - public MapWritable regionServerStartup(HServerInfo info, - long serverCurrentTime) throws IOException; - - /** - * Called to renew lease, tell master what the region server is doing and to - * receive new instructions from the master - * - * @param info server's address and start code - * @param msgs things the region server wants to tell the master - * @param mostLoadedRegions Array of HRegionInfos that should contain the - * reporting server's most loaded regions. These are candidates for being - * rebalanced. - * @return instructions from the master to the region server - * @throws IOException e - */ - public HMsg[] regionServerReport(HServerInfo info, HMsg msgs[], - HRegionInfo mostLoadedRegions[]) + public MapWritable regionServerStartup(final int port, + final long serverStartcode, final long serverCurrentTime) throws IOException; } \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java index be8abc1..e032491 100644 --- a/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java +++ b/src/main/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java @@ -290,7 +290,8 @@ public interface HRegionInterface extends VersionedProtocol, Stoppable, Abortabl /** * Method used when a master is taking the place of another failed one. - * @return The HSI + * @return This servers {@link HServerInfo}; it has RegionServer POV on the + * hostname which may not agree w/ how the Master sees this server. * @throws IOException e */ public HServerInfo getHServerInfo() throws IOException; diff --git a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 3f3b696..614e71a 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -37,7 +37,6 @@ import org.apache.hadoop.hbase.Chore; import org.apache.hadoop.hbase.ClusterStatus; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HMsg; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; import org.apache.hadoop.hbase.HServerInfo; @@ -345,7 +344,7 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server { this.connection = HConnectionManager.getConnection(conf); this.executorService = new ExecutorService(getServerName()); - this.serverManager = new ServerManager(this, this, metrics); + this.serverManager = new ServerManager(this, this); this.catalogTracker = new CatalogTracker(this.zooKeeper, this.connection, this, conf.getInt("hbase.master.catalog.timeout", Integer.MAX_VALUE)); @@ -601,25 +600,22 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server { } @Override - public MapWritable regionServerStartup(final HServerInfo serverInfo, - final long serverCurrentTime) + public MapWritable regionServerStartup(final int port, + final long serverStartCode, final long serverCurrentTime) throws IOException { - // Set the ip into the passed in serverInfo. Its ip is more than likely - // not the ip that the master sees here. See at end of this method where - // we pass it back to the regionserver by setting "hbase.regionserver.address" - // Everafter, the HSI combination 'server name' is what uniquely identifies - // the incoming RegionServer. - InetSocketAddress address = new InetSocketAddress( - HBaseServer.getRemoteIp().getHostName(), - serverInfo.getServerAddress().getPort()); - serverInfo.setServerAddress(new HServerAddress(address)); - + // This call to InetSocketAddress will resolve the hostname to an IP. + InetSocketAddress rsAddress = new InetSocketAddress( + HBaseServer.getRemoteIp().getHostName(), port); + if (rsAddress.isUnresolved()) { + LOG.warn("Failed resolve of " + rsAddress); + } // Register with server manager - this.serverManager.regionServerStartup(serverInfo, serverCurrentTime); + this.serverManager.regionServerStartup(rsAddress, serverStartCode, + serverCurrentTime); // Send back some config info MapWritable mw = createConfigurationSubset(); - mw.put(new Text("hbase.regionserver.address"), - serverInfo.getServerAddress()); + mw.put(new Text(HConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER), + new Text(rsAddress.getHostName())); return mw; } @@ -637,26 +633,6 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server { return mw; } - @Override - public HMsg [] regionServerReport(HServerInfo serverInfo, HMsg msgs[], - HRegionInfo[] mostLoadedRegions) - throws IOException { - return adornRegionServerAnswer(serverInfo, - this.serverManager.regionServerReport(serverInfo, msgs, mostLoadedRegions)); - } - - /** - * Override if you'd add messages to return to regionserver hsi - * or to send an exception. - * @param msgs Messages to add to - * @return Messages to return to - * @throws IOException exceptions that were injected for the region servers - */ - protected HMsg [] adornRegionServerAnswer(final HServerInfo hsi, - final HMsg [] msgs) throws IOException { - return msgs; - } - public boolean isMasterRunning() { return !isStopped(); } @@ -768,11 +744,11 @@ implements HMasterInterface, HMasterRegionInterface, MasterServices, Server { this.assignmentManager.unassign(hri); } else { dest = this.serverManager.getServerInfo(new String(destServerName)); - if (this.cpHost != null) { this.cpHost.preMove(p.getFirst(), p.getSecond(), dest); } RegionPlan rp = new RegionPlan(p.getFirst(), p.getSecond(), dest); + LOG.info("Added move plan " + rp + ", running balancer"); this.assignmentManager.balance(rp); if (this.cpHost != null) { this.cpHost.postMove(p.getFirst(), p.getSecond(), dest); diff --git a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java index 4d921da..b94adfe 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java +++ b/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.master; import java.io.IOException; +import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -27,12 +28,13 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListSet; +import org.apache.commons.lang.NotImplementedException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClockOutOfSyncException; -import org.apache.hadoop.hbase.HMsg; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; import org.apache.hadoop.hbase.HServerInfo; @@ -70,9 +72,9 @@ public class ServerManager { // Set if we are to shutdown the cluster. private volatile boolean clusterShutdown = false; - /** The map of known server names to server info */ - private final Map onlineServers = - new ConcurrentHashMap(); + /** Map of registered servers */ + private final Map onlineServers = + new ConcurrentHashMap(); // TODO: This is strange to have two maps but HSI above is used on both sides /** @@ -84,9 +86,6 @@ public class ServerManager { private final Server master; private final MasterServices services; - // Reporting to track master metrics. - private final MasterMetrics metrics; - private final DeadServer deadservers; private final long maxSkew; @@ -97,11 +96,9 @@ public class ServerManager { * @param services * @param metrics */ - public ServerManager(final Server master, final MasterServices services, - MasterMetrics metrics) { + public ServerManager(final Server master, final MasterServices services) { this.master = master; this.services = services; - this.metrics = metrics; Configuration c = master.getConfiguration(); maxSkew = c.getLong("hbase.master.maxclockskew", 30000); this.deadservers = @@ -110,11 +107,13 @@ public class ServerManager { /** * Let the server manager know a new regionserver has come online - * @param serverInfo + * @param address + * @param serverStartcode * @param serverCurrentTime The current time of the region server in ms * @throws IOException */ - void regionServerStartup(final HServerInfo serverInfo, long serverCurrentTime) + void regionServerStartup(final InetSocketAddress address, + final long serverStartcode, long serverCurrentTime) throws IOException { // Test for case where we get a region startup message from a regionserver // that has been quickly restarted but whose znode expiration handler has @@ -123,41 +122,48 @@ public class ServerManager { // is, reject the server and trigger its expiration. The next time it comes // in, it should have been removed from serverAddressToServerInfo and queued // for processing by ProcessServerShutdown. - HServerInfo info = new HServerInfo(serverInfo); - checkIsDead(info.getServerName(), "STARTUP"); - checkAlreadySameHostPort(info); - checkClockSkew(info, serverCurrentTime); - recordNewServer(info, false, null); + String serverName = HServerInfo.getServerName(address.getHostName(), + address.getPort(), serverStartcode); + checkIsDead(serverName, "STARTUP"); + checkAlreadySameHostPort(address, serverStartcode); + checkClockSkew(serverName, serverCurrentTime); + recordNewServer(serverName, address); } /** * Test to see if we have a server of same host and port already. - * @param serverInfo + * @param hostAndPort + * @param serverStartcode * @throws PleaseHoldException */ - void checkAlreadySameHostPort(final HServerInfo serverInfo) + void checkAlreadySameHostPort(final InetSocketAddress address, + final long serverStartcode) throws PleaseHoldException { - String hostAndPort = serverInfo.getServerAddress().toString(); - HServerInfo existingServer = - haveServerWithSameHostAndPortAlready(serverInfo.getHostnamePort()); + String existingServer = haveServerWithSameHostAndPortAlready(address); if (existingServer != null) { - String message = "Server start rejected; we already have " + hostAndPort + - " registered; existingServer=" + existingServer + ", newServer=" + serverInfo; + String message = "Server address=" + address + + ", startcode=" + serverStartcode + + " rejected; we already have " + existingServer + + " registered with same hostname and port"; LOG.info(message); - if (existingServer.getStartCode() < serverInfo.getStartCode()) { + long existingServerStartCode = + HServerInfo.getServerStartCodeFromServerName(existingServer); + if (existingServerStartCode < serverStartcode) { LOG.info("Triggering server recovery; existingServer " + - existingServer.getServerName() + " looks stale"); + existingServer + " looks stale"); expireServer(existingServer); } throw new PleaseHoldException(message); } } - private HServerInfo haveServerWithSameHostAndPortAlready(final String hostnamePort) { + private String haveServerWithSameHostAndPortAlready(final InetSocketAddress address) { synchronized (this.onlineServers) { - for (Map.Entry e: this.onlineServers.entrySet()) { - if (e.getValue().getHostnamePort().equals(hostnamePort)) { - return e.getValue(); + for (Map.Entry e: + this.onlineServers.entrySet()) { + if (e.getValue().getHostName().equals(address.getHostName()) && + e.getValue().getPort() == address.getPort()) { + return e.getKey(); } } } @@ -167,14 +173,16 @@ public class ServerManager { /** * Checks if the clock skew between the server and the master. If the clock * skew is too much it will throw an Exception. + * @param serverName Incoming servers's name + * @param serverCurrentTime * @throws ClockOutOfSyncException */ - private void checkClockSkew(final HServerInfo serverInfo, + private void checkClockSkew(final String serverName, final long serverCurrentTime) throws ClockOutOfSyncException { long skew = System.currentTimeMillis() - serverCurrentTime; if (skew > maxSkew) { - String message = "Server " + serverInfo.getServerName() + " has been " + + String message = "Server " + serverName + " has been " + "rejected; Reported time is too far out of sync with master. " + "Time difference of " + skew + "ms > max allowed of " + maxSkew + "ms"; LOG.warn(message); @@ -199,160 +207,13 @@ public class ServerManager { /** * Adds the HSI to the RS list - * @param info The region server informations - * @param useInfoLoad True if the load from the info should be used; e.g. - * under a master failover + * @param serverName The remote servers name. * @param hri Region interface. Can be null. */ - void recordNewServer(HServerInfo info, boolean useInfoLoad, - HRegionInterface hri) { - HServerLoad load = useInfoLoad? info.getLoad(): new HServerLoad(); - String serverName = info.getServerName(); - LOG.info("Registering server=" + serverName + ", regionCount=" + - load.getLoad() + ", userLoad=" + useInfoLoad); - info.setLoad(load); - // TODO: Why did we update the RS location ourself? Shouldn't RS do this? - // masterStatus.getZooKeeper().updateRSLocationGetWatch(info, watcher); - // -- If I understand the question, the RS does not update the location - // because could be disagreement over locations because of DNS issues; only - // master does DNS now -- St.Ack 20100929. - this.onlineServers.put(serverName, info); - if (hri == null) { - serverConnections.remove(serverName); - } else { - serverConnections.put(serverName, hri); - } - } - - /** - * Called to process the messages sent from the region server to the master - * along with the heart beat. - * - * @param serverInfo - * @param msgs - * @param mostLoadedRegions Array of regions the region server is submitting - * as candidates to be rebalanced, should it be overloaded - * @return messages from master to region server indicating what region - * server should do. - * - * @throws IOException - */ - HMsg [] regionServerReport(final HServerInfo serverInfo, - final HMsg [] msgs, final HRegionInfo[] mostLoadedRegions) - throws IOException { - // Be careful. This method does returns in the middle. - HServerInfo info = new HServerInfo(serverInfo); - - // Check if dead. If it is, it'll get a 'You Are Dead!' exception. - checkIsDead(info.getServerName(), "REPORT"); - - // If we don't know this server, tell it shutdown. - HServerInfo storedInfo = this.onlineServers.get(info.getServerName()); - if (storedInfo == null) { - // Maybe we already have this host+port combo and its just different - // start code? - checkAlreadySameHostPort(info); - // Just let the server in. Presume master joining a running cluster. - // recordNewServer is what happens at the end of reportServerStartup. - // The only thing we are skipping is passing back to the regionserver - // the HServerInfo to use. Here we presume a master has already done - // that so we'll press on with whatever it gave us for HSI. - recordNewServer(info, true, null); - // If msgs, put off their processing but this is not enough because - // its possible that the next time the server reports in, we'll still - // not be up and serving. For example, if a split, we'll need the - // regions and servers setup in the master before the below - // handleSplitReport will work. TODO: FIx!! - if (msgs.length > 0) - throw new PleaseHoldException("FIX! Putting off " + - "message processing because not yet rwady but possible we won't be " + - "ready next on next report"); - } - - // Check startcodes - if (raceThatShouldNotHappenAnymore(storedInfo, info)) { - return HMsg.STOP_REGIONSERVER_ARRAY; - } - - for (HMsg msg: msgs) { - LOG.info("Received " + msg + " from " + serverInfo.getServerName()); - switch (msg.getType()) { - case REGION_SPLIT: - this.services.getAssignmentManager().handleSplitReport(serverInfo, - msg.getRegionInfo(), msg.getDaughterA(), msg.getDaughterB()); - break; - - default: - LOG.error("Unhandled msg type " + msg); - } - } - - HMsg [] reply = null; - int numservers = countOfRegionServers(); - if (this.clusterShutdown) { - if (numservers <= 2) { - // Shutdown needs to be staggered; the meta regions need to close last - // in case they need to be updated during the close melee. If <= 2 - // servers left, then these are the two that were carrying root and meta - // most likely (TODO: This presumes unsplittable meta -- FIX). Tell - // these servers can shutdown now too. - reply = HMsg.STOP_REGIONSERVER_ARRAY; - } - } - return processRegionServerAllsWell(info, mostLoadedRegions, reply); - } - - private boolean raceThatShouldNotHappenAnymore(final HServerInfo storedInfo, - final HServerInfo reportedInfo) { - if (storedInfo.getStartCode() != reportedInfo.getStartCode()) { - // TODO: I don't think this possible any more. We check startcodes when - // server comes in on regionServerStartup -- St.Ack - // This state is reachable if: - // 1) RegionServer A started - // 2) RegionServer B started on the same machine, then clobbered A in regionServerStartup. - // 3) RegionServer A returns, expecting to work as usual. - // The answer is to ask A to shut down for good. - LOG.warn("Race condition detected: " + reportedInfo.getServerName()); - synchronized (this.onlineServers) { - removeServerInfo(reportedInfo.getServerName()); - notifyOnlineServers(); - } - return true; - } - return false; - } - - /** - * RegionServer is checking in, no exceptional circumstances - * @param serverInfo - * @param mostLoadedRegions - * @param msgs - * @return - * @throws IOException - */ - private HMsg[] processRegionServerAllsWell(HServerInfo serverInfo, - final HRegionInfo[] mostLoadedRegions, HMsg[] msgs) - throws IOException { - // Refresh the info object and the load information - this.onlineServers.put(serverInfo.getServerName(), serverInfo); - HServerLoad load = serverInfo.getLoad(); - if (load != null && this.metrics != null) { - this.metrics.incrementRequests(load.getNumberOfRequests()); - } - // No more piggyback messages on heartbeats for other stuff - return msgs; - } - - /** - * @param serverName - * @return True if we removed server from the list. - */ - private boolean removeServerInfo(final String serverName) { - HServerInfo info = this.onlineServers.remove(serverName); - if (info != null) { - return true; - } - return false; + void recordNewServer(final String serverName, final InetSocketAddress address) { + LOG.info("Registering server=" + serverName); + this.onlineServers.put(serverName, address); + this.serverConnections.remove(serverName); } /** @@ -362,6 +223,7 @@ public class ServerManager { * @return the average load */ public double getAverageLoad() { + /* TODO int totalLoad = 0; int numServers = 0; double averageLoad = 0.0; @@ -371,6 +233,8 @@ public class ServerManager { } averageLoad = (double)totalLoad / (double)numServers; return averageLoad; + */ + throw new NotImplementedException("TODO"); } /** @return the count of active regionservers */ @@ -380,17 +244,9 @@ public class ServerManager { } /** - * @param name server name - * @return HServerInfo for the given server address - */ - public HServerInfo getServerInfo(String name) { - return this.onlineServers.get(name); - } - - /** * @return Read-only map of servers to serverinfo */ - public Map getOnlineServers() { + public Map getOnlineServers() { // Presumption is that iterating the returned Map is OK. synchronized (this.onlineServers) { return Collections.unmodifiableMap(this.onlineServers); @@ -426,12 +282,6 @@ public class ServerManager { return null; } - private void notifyOnlineServers() { - synchronized (this.onlineServers) { - this.onlineServers.notifyAll(); - } - } - /* * Wait on regionservers to report in * with {@link #regionServerReport(HServerInfo, HMsg[])} so they get notice @@ -462,19 +312,16 @@ public class ServerManager { * Expire the passed server. Add it to list of deadservers and queue a * shutdown processing. */ - public synchronized void expireServer(final HServerInfo hsi) { - // First check a server to expire. ServerName is of the form: - // , , - String serverName = hsi.getServerName(); - HServerInfo info = this.onlineServers.get(serverName); - if (info == null) { - LOG.warn("Received expiration of " + hsi.getServerName() + + public synchronized void expireServer(final String serverName) { + InetSocketAddress serverAddress = this.onlineServers.get(serverName); + if (serverAddress == null) { + LOG.warn("Received expiration of " + serverName + " but server is not currently online"); return; } if (this.deadservers.contains(serverName)) { // TODO: Can this happen? It shouldn't be online in this case? - LOG.warn("Received expiration of " + hsi.getServerName() + + LOG.warn("Received expiration of " + serverName + " but server shutdown is already in progress"); return; } @@ -487,7 +334,7 @@ public class ServerManager { // If cluster is going down, yes, servers are going to be expiring; don't // process as a dead server if (this.clusterShutdown) { - LOG.info("Cluster shutdown set; " + hsi.getServerName() + + LOG.info("Cluster shutdown set; " + serverName + " expired; onlineServers=" + this.onlineServers.size()); if (this.onlineServers.isEmpty()) { master.stop("Cluster shutdown set; onlineServer=0"); @@ -499,8 +346,7 @@ public class ServerManager { boolean carryingRoot; try { HServerAddress address = ct.getRootLocation(); - carryingRoot = address != null && - hsi.getServerAddress().equals(address); + carryingRoot = isSameAddress(address, serverAddress); } catch (InterruptedException e) { Thread.currentThread().interrupt(); LOG.info("Interrupted"); @@ -512,20 +358,24 @@ public class ServerManager { // has an inmemory list of who has what. This list will be cleared as we // process the dead server but should be find asking it now. HServerAddress address = ct.getMetaLocation(); - boolean carryingMeta = - address != null && hsi.getServerAddress().equals(address); + boolean carryingMeta = isSameAddress(address, serverAddress); if (carryingRoot || carryingMeta) { this.services.getExecutorService().submit(new MetaServerShutdownHandler(this.master, - this.services, this.deadservers, info, carryingRoot, carryingMeta)); + this.services, this.deadservers, serverName, carryingRoot, carryingMeta)); } else { this.services.getExecutorService().submit(new ServerShutdownHandler(this.master, - this.services, this.deadservers, info)); + this.services, this.deadservers, serverName)); } LOG.debug("Added=" + serverName + " to dead servers, submitted shutdown handler to be executed, root=" + carryingRoot + ", meta=" + carryingMeta); } + private static boolean isSameAddress(final HServerAddress hsa, + final InetSocketAddress serverAddress) { + return hsa != null && hsa.getInetSocketAddress().equals(serverAddress); + } + // RPC methods to region servers /** diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java index eb01a6a..3bc5fba 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java +++ b/src/main/java/org/apache/hadoop/hbase/master/handler/MetaServerShutdownHandler.java @@ -34,9 +34,9 @@ public class MetaServerShutdownHandler extends ServerShutdownHandler { public MetaServerShutdownHandler(final Server server, final MasterServices services, - final DeadServer deadServers, final HServerInfo hsi, + final DeadServer deadServers, final String serverName, final boolean carryingRoot, final boolean carryingMeta) { - super(server, services, deadServers, hsi, EventType.M_META_SERVER_SHUTDOWN); + super(server, services, deadServers, serverName, EventType.M_META_SERVER_SHUTDOWN); this.carryingRoot = carryingRoot; this.carryingMeta = carryingMeta; } diff --git a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java index 852efe7..0074903 100644 --- a/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java +++ b/src/main/java/org/apache/hadoop/hbase/master/handler/ServerShutdownHandler.java @@ -51,25 +51,25 @@ import org.apache.zookeeper.KeeperException; */ public class ServerShutdownHandler extends EventHandler { private static final Log LOG = LogFactory.getLog(ServerShutdownHandler.class); - private final HServerInfo hsi; + private final String serverName; private final Server server; private final MasterServices services; private final DeadServer deadServers; public ServerShutdownHandler(final Server server, final MasterServices services, - final DeadServer deadServers, final HServerInfo hsi) { + final DeadServer deadServers, final String serverName) { this(server, services, deadServers, hsi, EventType.M_SERVER_SHUTDOWN); } ServerShutdownHandler(final Server server, final MasterServices services, - final DeadServer deadServers, final HServerInfo hsi, EventType type) { + final DeadServer deadServers, final String serverName, EventType type) { super(server, type); - this.hsi = hsi; + this.serverName = serverName; this.server = server; this.services = services; this.deadServers = deadServers; - if (!this.deadServers.contains(hsi.getServerName())) { - LOG.warn(hsi.getServerName() + " is NOT in deadservers; it should be!"); + if (!this.deadServers.contains(this.serverName)) { + LOG.warn(this.serverName + " is NOT in deadservers; it should be!"); } } @@ -89,7 +89,7 @@ public class ServerShutdownHandler extends EventHandler { @Override public void process() throws IOException { - final String serverName = this.hsi.getServerName(); + final String serverName = this.serverName; LOG.info("Splitting logs for " + serverName); this.services.getMasterFileSystem().splitLog(serverName); diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 6f80fb0..13be5de 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -58,12 +58,12 @@ import org.apache.hadoop.hbase.DroppedSnapshotException; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HConstants.OperationStatusCode; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.UnknownScannerException; -import org.apache.hadoop.hbase.HConstants.OperationStatusCode; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Increment; @@ -93,7 +93,6 @@ import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Writables; import org.apache.hadoop.io.Writable; -import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.StringUtils; import com.google.common.collect.ClassToInstanceMap; @@ -229,7 +228,7 @@ public class HRegion implements HeapSize { // , Writable{ final long memstoreFlushSize; private volatile long lastFlushTime; final RegionServerServices rsServices; - private List> recentFlushes = new ArrayList>(); + private List> recentFlushes = new ArrayList>(); private final long blockingMemStoreSize; final long threadWakeFrequency; // Used to guard closes diff --git a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index b3f47d4..5397630 100644 --- a/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -50,6 +50,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.commons.lang.NotImplementedException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -61,7 +62,6 @@ import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HConstants.OperationStatusCode; -import org.apache.hadoop.hbase.HMsg; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; import org.apache.hadoop.hbase.HServerInfo; @@ -71,6 +71,7 @@ import org.apache.hadoop.hbase.MasterAddressTracker; import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.RemoteExceptionHandler; import org.apache.hadoop.hbase.Server; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.UnknownRowLockException; import org.apache.hadoop.hbase.UnknownScannerException; @@ -165,7 +166,13 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, // If false, the file system has become unavailable protected volatile boolean fsOk; + // Holds startcode, webui port, and server address. protected HServerInfo serverInfo; + // Gets set after we register w/ Master; has the hostname that the master sees + // for us rather than what we found ourselves locally. Use this editing + // .META., etc. We used to get serverName from HServerInfo but was unreliable. + private String serverName = null; + protected final Configuration conf; private final HConnection connection; @@ -182,7 +189,6 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, new HashMap(); protected final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - private final LinkedBlockingQueue outboundMsgs = new LinkedBlockingQueue(); final int numRetries; protected final int threadWakeFrequency; @@ -241,7 +247,8 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, // flag set after we're done setting up server threads (used for testing) protected volatile boolean isOnline; - final Map scanners = new ConcurrentHashMap(); + final Map scanners = + new ConcurrentHashMap(); // zookeeper connection and watcher private ZooKeeperWatcher zooKeeper; @@ -260,10 +267,6 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, private final int rpcTimeout; - // The main region server thread. - @SuppressWarnings("unused") - private Thread regionServerThread; - // Instance of the hbase executor service. private ExecutorService service; @@ -317,33 +320,29 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, this.abortRequested = false; this.stopped = false; - // Server to handle client requests - String machineName = DNS.getDefaultHost(conf.get( - "hbase.regionserver.dns.interface", "default"), conf.get( - "hbase.regionserver.dns.nameserver", "default")); - String addressStr = machineName + ":" + - conf.get(HConstants.REGIONSERVER_PORT, - Integer.toString(HConstants.DEFAULT_REGIONSERVER_PORT)); - HServerAddress address = new HServerAddress(addressStr); + // Server to handle client requests. + String hostname = DNS.getDefaultHost( + conf.get("hbase.regionserver.dns.interface", "default"), + conf.get("hbase.regionserver.dns.nameserver", "default")); + int port = conf.getInt(HConstants.REGIONSERVER_PORT, + HConstants.DEFAULT_REGIONSERVER_PORT); + HServerAddress address = new HServerAddress(hostname, port); this.server = HBaseRPC.getServer(this, - new Class[]{HRegionInterface.class, HBaseRPCErrorHandler.class, + new Class[]{HRegionInterface.class, HBaseRPCErrorHandler.class, OnlineRegions.class}, - address.getBindAddress(), - address.getPort(), conf.getInt("hbase.regionserver.handler.count", 10), + address.getBindAddress(), // BindAddress is IP we got for this server. + address.getPort(), + conf.getInt("hbase.regionserver.handler.count", 10), conf.getInt("hbase.regionserver.metahandler.count", 10), false, conf, QOS_THRESHOLD); + // Recreate address in case server changed anything. + address = new HServerAddress(this.server.getListenerAddress()); this.server.setErrorHandler(this); this.server.setQosFunction(new QosFunction()); - - // HServerInfo can be amended by master. See below in reportForDuty. - this.serverInfo = new HServerInfo(new HServerAddress(new InetSocketAddress( - address.getBindAddress(), this.server.getListenerAddress().getPort())), - System.currentTimeMillis(), this.conf.getInt( - "hbase.regionserver.info.port", 60030), machineName); - if (this.serverInfo.getServerAddress() == null) { - throw new NullPointerException("Server address cannot be null; " - + "hbase-958 debugging"); - } + // HServerInfo holds address, webui, and startcode. + int webuiPort = putPutUpWebUI(); + this.serverInfo = new HServerInfo(address, System.currentTimeMillis(), + this.conf.getInt("hbase.regionserver.info.port", webuiPort)); } private static final int NORMAL_QOS = 0; @@ -479,8 +478,8 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, */ private void initializeZooKeeper() throws IOException, InterruptedException { // Open connection to zookeeper and set primary watcher - zooKeeper = new ZooKeeperWatcher(conf, REGIONSERVER + ":" + - serverInfo.getServerAddress().getPort(), this); + this.zooKeeper = new ZooKeeperWatcher(conf, REGIONSERVER + ":" + + this.serverInfo.getServerAddress().getPort(), this); // Create the master address manager, register with zk, and start it. Then // block until a master is available. No point in starting up if no master @@ -525,7 +524,6 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, } private void initializeThreads() throws IOException { - // Cache flushing thread. this.cacheFlusher = new MemStoreFlusher(conf, this); @@ -534,10 +532,10 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, // Background thread to check for major compactions; needed if region // has not gotten updates in a while. Make it run at a lesser frequency. - int multiplier = this.conf.getInt(HConstants.THREAD_WAKE_FREQUENCY - + ".multiplier", 1000); + int multiplier = this.conf.getInt(HConstants.THREAD_WAKE_FREQUENCY + + ".multiplier", 1000); this.majorCompactionChecker = new MajorCompactionChecker(this, - this.threadWakeFrequency * multiplier, this); + this.threadWakeFrequency * multiplier, this); this.leases = new Leases((int) conf.getLong( HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY, @@ -551,21 +549,22 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, * load/unload instructions. */ public void run() { - try { - // Initialize threads and wait for a master + // Do pre-registration initializations; zookeeper, lease threads, etc. initialize(); } catch (Exception e) { abort("Fatal exception during initialization", e); - } - this.regionServerThread = Thread.currentThread(); try { + // Try and register with the Master; tell it we are here. while (!this.stopped) { if (tryReportForDuty()) break; + LOG.warn("No response on reportForDuty. Sleeping and then retrying."); + this.sleeper.sleep(); } + + // We registered with the Master. Go into run mode. long lastMsg = 0; - List outboundMessages = new ArrayList(); // The main run loop. for (int tries = 0; !this.stopped && isHealthy();) { if (!isClusterUp()) { @@ -794,61 +793,25 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, } /* - * Add to the passed msgs messages to pass to the master. - * - * @param msgs Current outboundMsgs array; we'll add messages to this List. - */ - private void addOutboundMsgs(final List msgs) { - if (msgs.isEmpty()) { - this.outboundMsgs.drainTo(msgs); - return; - } - OUTER: for (HMsg m : this.outboundMsgs) { - for (HMsg mm : msgs) { - // Be careful don't add duplicates. - if (mm.equals(m)) { - continue OUTER; - } - } - msgs.add(m); - } - } - - /* - * Remove from this.outboundMsgs those messsages we sent the master. - * - * @param msgs Messages we sent the master. - */ - private void updateOutboundMsgs(final List msgs) { - if (msgs.isEmpty()) { - return; - } - for (HMsg m : this.outboundMsgs) { - for (HMsg mm : msgs) { - if (mm.equals(m)) { - this.outboundMsgs.remove(m); - break; - } - } - } - } - - /* * Run init. Sets up hlog and starts up all server threads. * * @param c Extra configuration. */ - protected void handleReportForDutyResponse(final MapWritable c) throws IOException { + protected void handleReportForDutyResponse(final MapWritable c) + throws IOException { try { for (Map.Entry e : c.entrySet()) { - String key = e.getKey().toString(); // Use the address the master passed us - if (key.equals("hbase.regionserver.address")) { - HServerAddress hsa = (HServerAddress) e.getValue(); - LOG.info("Master passed us address to use. Was=" - + this.serverInfo.getServerAddress() + ", Now=" + hsa.toString()); - this.serverInfo.setServerAddress(hsa); + if (key.equals(HConstants.KEY_FOR_HOSTNAME_SEEN_BY_MASTER)) { + String hostname = e.getValue().toString(); + LOG.info("Master passed us hostname to use. Was=" + + this.serverInfo.getServerAddress().getHostname() + ", Now=" + + hostname); + // this.serverName is how the master sees us. + this.serverName = ServerName.getServerName(hostname, + this.serverInfo.getServerAddress().getPort(), + this.serverInfo.getStartCode()); continue; } String value = e.getValue().toString(); @@ -857,13 +820,13 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, } this.conf.set(key, value); } - + // hack! Maps DFSClient => RegionServer for logs. HDFS made this // config param for task trackers, but we can piggyback off of it. if (this.conf.get("mapred.task.id") == null) { this.conf.set("mapred.task.id", - "hb_rs_" + this.serverInfo.getServerName() + "_" + - System.currentTimeMillis()); + "hb_rs_" + this.serverName.toString() + "_" + + System.currentTimeMillis()); } // Master sent us hbase.rootdir to use. Should be fully qualified @@ -878,8 +841,16 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, this.hlog = setupWALAndReplication(); // Init in here rather than in constructor after thread name has been set this.metrics = new RegionServerMetrics(); + + + HERE!!!!! + startServiceThreads(); - LOG.info("Serving as " + this.serverInfo.getServerName() + + String znodePath = + ZKUtil.joinZNode(this.zooKeeper.rsZNode, this.serverName.toString()); + ZKUtil.setAddressAndWatch(this.zooKeeper, znodePath, + this.serverInfo.getServerAddress()); + LOG.info("Serving as " + this.serverName + ", RPC listening on " + this.server.getListenerAddress() + ", sessionid=0x" + Long.toHexString(this.zooKeeper.getZooKeeper().getSessionId())); @@ -1251,35 +1222,13 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, // an unhandled exception, it will just exit. this.leases.setName(n + ".leaseChecker"); this.leases.start(); - // Put up info server. - int port = this.conf.getInt("hbase.regionserver.info.port", 60030); - // -1 is for disabling info server - if (port >= 0) { - String addr = this.conf.get("hbase.regionserver.info.bindAddress", - "0.0.0.0"); - // check if auto port bind enabled - boolean auto = this.conf.getBoolean("hbase.regionserver.info.port.auto", - false); - while (true) { - try { - this.infoServer = new InfoServer("regionserver", addr, port, false); - this.infoServer.setAttribute("regionserver", this); - this.infoServer.start(); - break; - } catch (BindException e) { - if (!auto) { - // auto bind disabled throw BindException - throw e; - } - // auto bind enabled, try to use another port - LOG.info("Failed binding http info server to port: " + port); - port++; - // update HRS server info port. - this.serverInfo = new HServerInfo(this.serverInfo.getServerAddress(), - this.serverInfo.getStartCode(), port, - this.serverInfo.getHostname()); - } - } + + // Put up the webui. Webui may come up on port other than configured if + // that port is occupied. Adjust serverInfo if this is the case. + int webuiport = putPutUpWebUI(); + if (this.serverInfo.getInfoPort() != webuiport) { + this.serverInfo = new HServerInfo(this.serverInfo.getServerAddress(), + this.serverInfo.getStartCode(), webuiport); } if (this.replicationHandler != null) { @@ -1291,6 +1240,37 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, this.server.start(); } + /** + * Puts up the webui. + * @return Returns final port -- maybe different from what we started with. + * @throws IOException + */ + private int putPutUpWebUI() throws IOException { + int port = this.conf.getInt("hbase.regionserver.info.port", 60030); + // -1 is for disabling info server + if (port < 0) return port; + String addr = this.conf.get("hbase.regionserver.info.bindAddress", "0.0.0.0"); + // check if auto port bind enabled + boolean auto = this.conf.getBoolean("hbase.regionserver.info.port.auto", false); + while (true) { + try { + this.infoServer = new InfoServer("regionserver", addr, port, false); + this.infoServer.setAttribute("regionserver", this); + this.infoServer.start(); + break; + } catch (BindException e) { + if (!auto) { + // auto bind disabled throw BindException + throw e; + } + // auto bind enabled, try to use another port + LOG.info("Failed binding http info server to port: " + port); + port++; + } + } + return port; + } + /* * Verify that server is healthy */ @@ -1475,35 +1455,32 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, handleReportForDutyResponse(w); return true; } - sleeper.sleep(); - LOG.warn("No response on reportForDuty. Sleeping and then retrying."); return false; } /* * Let the master know we're here Run initialization using parameters passed * us by the master. + * @return A Map of key/value configurations we got from the Master else + * null if we failed to register. + * @throws IOException */ private MapWritable reportForDuty() throws IOException { HServerAddress masterAddress = null; while (!stopped && (masterAddress = getMaster()) == null) { + LOG.warn("Unable to get master for initialization -- sleeping"); sleeper.sleep(); - LOG.warn("Unable to get master for initialization"); } - MapWritable result = null; long lastMsg = 0; while (!stopped) { try { this.requestCount.set(0); - lastMsg = System.currentTimeMillis(); - ZKUtil.setAddressAndWatch(zooKeeper, - ZKUtil.joinZNode(zooKeeper.rsZNode, ZKUtil.getNodeName(serverInfo)), - this.serverInfo.getServerAddress()); - this.serverInfo.setLoad(buildServerLoad()); LOG.info("Telling master at " + masterAddress + " that we are up"); - result = this.hbaseMaster.regionServerStartup(this.serverInfo, - EnvironmentEdgeManager.currentTimeMillis()); + lastMsg = EnvironmentEdgeManager.currentTimeMillis(); + int port = this.serverInfo.getServerAddress().getPort(); + result = this.hbaseMaster.regionServerStartup(port, + this.serverInfo.getStartCode(), lastMsg); break; } catch (RemoteException e) { IOException ioe = e.unwrapRemoteException(); @@ -1517,8 +1494,6 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, } } catch (IOException e) { LOG.warn("error telling master we are up", e); - } catch (KeeperException e) { - LOG.warn("error putting up ephemeral node in zookeeper", e); } sleeper.sleep(lastMsg); } @@ -1537,11 +1512,14 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, */ void reportSplit(HRegionInfo oldRegion, HRegionInfo newRegionA, HRegionInfo newRegionB) { + throw new NotImplementedException("FIX"); + /* this.outboundMsgs.add(new HMsg( HMsg.Type.REGION_SPLIT, oldRegion, newRegionA, newRegionB, Bytes.toBytes("Daughters; " + newRegionA.getRegionNameAsString() + ", " + newRegionB.getRegionNameAsString()))); + */ } /** @@ -2457,13 +2435,6 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, } /** - * @return Queue to which you can add outbound messages. - */ - protected LinkedBlockingQueue getOutboundMsgs() { - return this.outboundMsgs; - } - - /** * Return the total size of all memstores in every region. * * @return memstore size in bytes @@ -2500,13 +2471,18 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, } /** - * @return Info on port this server has bound to, etc. + * @return This servers {@link HServerInfo} + * @deprecated Use {@link #getServerName()} instead. */ public HServerInfo getServerInfo() { - return this.serverInfo; + try { + return getHServerInfo(); + } catch (IOException e) { + e.printStackTrace(); + } + return null; } - @Override public Result increment(byte[] regionName, Increment increment) throws IOException { @@ -2582,11 +2558,13 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, } } - /** {@inheritDoc} */ + /** {@inheritDoc} + * @deprecated Use {@link #getServerName()} instead. + */ @Override @QosPriority(priority=HIGH_QOS) public HServerInfo getHServerInfo() throws IOException { - return serverInfo; + return this.serverInfo; } @SuppressWarnings("unchecked") @@ -2755,7 +2733,7 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, @Override public String getServerName() { - return serverInfo.getServerName(); + return this.serverName; } @Override diff --git a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java index ead223f..09dfd00 100644 --- a/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java +++ b/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java @@ -126,19 +126,6 @@ public class ZKUtil { } /** - * Get the unique node-name for the specified regionserver. - * - * Used when a server puts up an ephemeral node for itself and needs to use - * a unique name. - * - * @param serverInfo server information - * @return unique, zookeeper-safe znode path for the server instance - */ - public static String getNodeName(HServerInfo serverInfo) { - return serverInfo.getServerName(); - } - - /** * Get the name of the current node from the specified fully-qualified path. * @param path fully-qualified path * @return name of the current node diff --git a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java index 9ad3697..f5fba08 100644 --- a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java +++ b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java @@ -21,11 +21,8 @@ package org.apache.hadoop.hbase; import java.io.IOException; import java.security.PrivilegedAction; -import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -39,9 +36,7 @@ import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.Threads; -import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.io.MapWritable; -import org.apache.zookeeper.KeeperException; /** * This class creates a single process HBase cluster. @@ -86,75 +81,6 @@ public class MiniHBaseCluster { } /** - * Override Master so can add inject behaviors testing. - */ - public static class MiniHBaseClusterMaster extends HMaster { - private final Map> messages = - new ConcurrentHashMap>(); - - private final Map exceptions = - new ConcurrentHashMap(); - - public MiniHBaseClusterMaster(final Configuration conf) - throws IOException, KeeperException, InterruptedException { - super(conf); - } - - /** - * Add a message to send to a regionserver next time it checks in. - * @param hsi RegionServer's HServerInfo. - * @param msg Message to add. - */ - void addMessage(final HServerInfo hsi, HMsg msg) { - synchronized(this.messages) { - List hmsgs = this.messages.get(hsi); - if (hmsgs == null) { - hmsgs = new ArrayList(); - this.messages.put(hsi, hmsgs); - } - hmsgs.add(msg); - } - } - - void addException(final HServerInfo hsi, final IOException ex) { - this.exceptions.put(hsi, ex); - } - - /** - * This implementation is special, exceptions will be treated first and - * message won't be sent back to the region servers even if some are - * specified. - * @param hsi the rs - * @param msgs Messages to add to - * @return - * @throws IOException will be throw if any added for this region server - */ - @Override - protected HMsg[] adornRegionServerAnswer(final HServerInfo hsi, - final HMsg[] msgs) throws IOException { - IOException ex = this.exceptions.remove(hsi); - if (ex != null) { - throw ex; - } - HMsg [] answerMsgs = msgs; - synchronized (this.messages) { - List hmsgs = this.messages.get(hsi); - if (hmsgs != null && !hmsgs.isEmpty()) { - int size = answerMsgs.length; - HMsg [] newAnswerMsgs = new HMsg[size + hmsgs.size()]; - System.arraycopy(answerMsgs, 0, newAnswerMsgs, 0, answerMsgs.length); - for (int i = 0; i < hmsgs.size(); i++) { - newAnswerMsgs[answerMsgs.length + i] = hmsgs.get(i); - } - answerMsgs = newAnswerMsgs; - hmsgs.clear(); - } - } - return super.adornRegionServerAnswer(hsi, answerMsgs); - } - } - - /** * Subclass so can get at protected methods (none at moment). Also, creates * a FileSystem instance per instantiation. Adds a shutdown own FileSystem * on the way out. Shuts down own Filesystem only, not All filesystems as @@ -260,8 +186,7 @@ public class MiniHBaseCluster { try { // start up a LocalHBaseCluster hbaseCluster = new LocalHBaseCluster(conf, nMasterNodes, 0, - MiniHBaseCluster.MiniHBaseClusterMaster.class, - MiniHBaseCluster.MiniHBaseClusterRegionServer.class); + HMaster.class, MiniHBaseCluster.MiniHBaseClusterRegionServer.class); // manually add the regionservers as other users for (int i=0; i msgs = new ArrayList(); - HMsg hmsg = null; - final int size = 10; - for (int i = 0; i < size; i++) { - byte [] b = Bytes.toBytes(i); - hmsg = new HMsg(HMsg.Type.STOP_REGIONSERVER, - new HRegionInfo(new HTableDescriptor(Bytes.toBytes("test")), b, b)); - msgs.add(hmsg); - } - assertEquals(size, msgs.size()); - int index = msgs.indexOf(hmsg); - assertNotSame(-1, index); - msgs.remove(index); - assertEquals(size - 1, msgs.size()); - byte [] other = Bytes.toBytes("other"); - hmsg = new HMsg(HMsg.Type.STOP_REGIONSERVER, - new HRegionInfo(new HTableDescriptor(Bytes.toBytes("test")), other, other)); - assertEquals(-1, msgs.indexOf(hmsg)); - // Assert that two HMsgs are same if same content. - byte [] b = Bytes.toBytes(1); - hmsg = new HMsg(HMsg.Type.STOP_REGIONSERVER, - new HRegionInfo(new HTableDescriptor(Bytes.toBytes("test")), b, b)); - assertNotSame(-1, msgs.indexOf(hmsg)); - } - - public void testSerialization() throws IOException { - // Check out new HMsg that carries two daughter split regions. - byte [] abytes = Bytes.toBytes("a"); - byte [] bbytes = Bytes.toBytes("b"); - byte [] parentbytes = Bytes.toBytes("parent"); - HRegionInfo parent = - new HRegionInfo(new HTableDescriptor(Bytes.toBytes("parent")), - parentbytes, parentbytes); - // Assert simple HMsg serializes - HMsg hmsg = new HMsg(HMsg.Type.STOP_REGIONSERVER, parent); - byte [] bytes = Writables.getBytes(hmsg); - HMsg close = (HMsg)Writables.getWritable(bytes, new HMsg()); - assertTrue(close.equals(hmsg)); - // Assert split serializes - HRegionInfo daughtera = - new HRegionInfo(new HTableDescriptor(Bytes.toBytes("a")), abytes, abytes); - HRegionInfo daughterb = - new HRegionInfo(new HTableDescriptor(Bytes.toBytes("b")), bbytes, bbytes); - HMsg splithmsg = new HMsg(HMsg.Type.REGION_SPLIT, - parent, daughtera, daughterb, Bytes.toBytes("REGION_SPLIT")); - bytes = Writables.getBytes(splithmsg); - hmsg = (HMsg)Writables.getWritable(bytes, new HMsg()); - assertTrue(splithmsg.equals(hmsg)); - } -} diff --git a/src/test/java/org/apache/hadoop/hbase/TestHRegionLocation.java b/src/test/java/org/apache/hadoop/hbase/TestHRegionLocation.java new file mode 100644 index 0000000..0a6d895 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestHRegionLocation.java @@ -0,0 +1,66 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.junit.Test; + +public class TestHRegionLocation { + @Test + public void testHashAndEqualsCode() { + HServerAddress hsa1 = new HServerAddress("localhost", 1234); + HRegionLocation hrl1 = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, + hsa1); + HRegionLocation hrl2 = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, + hsa1); + assertEquals(hrl1.hashCode(), hrl2.hashCode()); + assertTrue(hrl1.equals(hrl2)); + HRegionLocation hrl3 = new HRegionLocation(HRegionInfo.ROOT_REGIONINFO, + hsa1); + assertNotSame(hrl1, hrl3); + assertFalse(hrl1.equals(hrl3)); + } + + @Test + public void testToString() { + HServerAddress hsa1 = new HServerAddress("localhost", 1234); + HRegionLocation hrl1 = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO, + hsa1); + System.out.println(hrl1.toString()); + } + + @Test + public void testCompareTo() { + HServerAddress hsa1 = new HServerAddress("localhost:1234"); + HRegionLocation hsl1 = new HRegionLocation(HRegionInfo.ROOT_REGIONINFO, hsa1); + HServerAddress hsa2 = new HServerAddress("localhost:1235"); + HRegionLocation hsl2 = new HRegionLocation(HRegionInfo.ROOT_REGIONINFO, hsa2); + assertTrue(hsl1.compareTo(hsl1) == 0); + assertTrue(hsl2.compareTo(hsl2) == 0); + int compare1 = hsl1.compareTo(hsl2); + int compare2 = hsl2.compareTo(hsl1); + assertTrue((compare1 > 0)? compare2 < 0: compare2 > 0); + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/TestHServerAddress.java b/src/test/java/org/apache/hadoop/hbase/TestHServerAddress.java new file mode 100644 index 0000000..1121ba4 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestHServerAddress.java @@ -0,0 +1,95 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.*; + +import java.io.IOException; +import java.net.InetSocketAddress; + +import org.apache.hadoop.hbase.util.Writables; +import org.junit.Test; + +/** + * Tests for {@link HServerAddress} + */ +public class TestHServerAddress { + @Test + public void testHashCode() { + HServerAddress hsa1 = new HServerAddress("localhost", 1234); + HServerAddress hsa2 = new HServerAddress("localhost", 1234); + assertEquals(hsa1.hashCode(), hsa2.hashCode()); + HServerAddress hsa3 = new HServerAddress("localhost", 1235); + assertNotSame(hsa1.hashCode(), hsa3.hashCode()); + } + + @Test + public void testHServerAddress() { + new HServerAddress(); + } + + @Test + public void testHServerAddressInetSocketAddress() { + HServerAddress hsa1 = + new HServerAddress(new InetSocketAddress("localhost", 1234)); + System.out.println(hsa1.toString()); + } + + @Test + public void testHServerAddressString() { + HServerAddress hsa1 = new HServerAddress("localhost:1234"); + HServerAddress hsa2 = + new HServerAddress(new InetSocketAddress("localhost", 1234)); + assertTrue(hsa1.equals(hsa2)); + } + + @Test (expected = IllegalArgumentException.class) + public void testHServerAddressString2() { + // Make address with wrong delimiter + new HServerAddress("localhost.1234"); + } + + @Test (expected = IllegalArgumentException.class) + public void testHServerAddressString3() { + // Make address with bad port + new HServerAddress("localhost:1234x"); + } + + @Test + public void testHServerAddressHServerAddress() { + HServerAddress hsa1 = new HServerAddress("localhost:1234"); + HServerAddress hsa2 = new HServerAddress(hsa1); + assertEquals(hsa1, hsa2); + } + + @Test + public void testReadFields() throws IOException { + HServerAddress hsa1 = new HServerAddress("localhost:1234"); + HServerAddress hsa2 = new HServerAddress("localhost:1235"); + byte [] bytes = Writables.getBytes(hsa1); + HServerAddress deserialized = + (HServerAddress)Writables.getWritable(bytes, new HServerAddress()); + assertEquals(hsa1, deserialized); + bytes = Writables.getBytes(hsa2); + deserialized = + (HServerAddress)Writables.getWritable(bytes, new HServerAddress()); + assertNotSame(hsa1, deserialized); + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/TestHServerInfo.java b/src/test/java/org/apache/hadoop/hbase/TestHServerInfo.java new file mode 100644 index 0000000..0504002 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestHServerInfo.java @@ -0,0 +1,80 @@ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.*; + +import java.io.IOException; + +import org.apache.hadoop.hbase.util.Writables; +import org.junit.Test; + +public class TestHServerInfo { + + @Test + public void testHashCodeAndEquals() { + HServerAddress hsa1 = new HServerAddress("localhost", 1234); + HServerInfo hsi1 = new HServerInfo(hsa1, 1L, 5678); + HServerInfo hsi2 = new HServerInfo(hsa1, 1L, 5678); + HServerInfo hsi3 = new HServerInfo(hsa1, 2L, 5678); + HServerInfo hsi4 = new HServerInfo(hsa1, 1L, 5677); + HServerAddress hsa2 = new HServerAddress("localhost", 1235); + HServerInfo hsi5 = new HServerInfo(hsa2, 1L, 5678); + assertEquals(hsi1.hashCode(), hsi2.hashCode()); + assertTrue(hsi1.equals(hsi2)); + assertNotSame(hsi1.hashCode(), hsi3.hashCode()); + assertFalse(hsi1.equals(hsi3)); + assertNotSame(hsi1.hashCode(), hsi4.hashCode()); + assertFalse(hsi1.equals(hsi4)); + assertNotSame(hsi1.hashCode(), hsi5.hashCode()); + assertFalse(hsi1.equals(hsi5)); + } + + @Test + public void testHServerInfoHServerInfo() { + HServerAddress hsa1 = new HServerAddress("localhost", 1234); + HServerInfo hsi1 = new HServerInfo(hsa1, 1L, 5678); + HServerInfo hsi2 = new HServerInfo(hsi1); + assertEquals(hsi1, hsi2); + } + + @Test + public void testGetServerAddress() { + HServerAddress hsa1 = new HServerAddress("localhost", 1234); + HServerInfo hsi1 = new HServerInfo(hsa1, 1L, 5678); + assertEquals(hsi1.getServerAddress(), hsa1); + } + + @Test + public void testToString() { + HServerAddress hsa1 = new HServerAddress("localhost", 1234); + HServerInfo hsi1 = new HServerInfo(hsa1, 1L, 5678); + System.out.println(hsi1.toString()); + } + + @Test + public void testReadFields() throws IOException { + HServerAddress hsa1 = new HServerAddress("localhost:1234"); + HServerInfo hsi1 = new HServerInfo(hsa1, 1L, 5678); + HServerAddress hsa2 = new HServerAddress("localhost:1235"); + HServerInfo hsi2 = new HServerInfo(hsa2, 1L, 5678); + byte [] bytes = Writables.getBytes(hsi1); + HServerInfo deserialized = + (HServerInfo)Writables.getWritable(bytes, new HServerInfo()); + assertEquals(hsi1, deserialized); + bytes = Writables.getBytes(hsi2); + deserialized = (HServerInfo)Writables.getWritable(bytes, new HServerInfo()); + assertNotSame(hsa1, deserialized); + } + + @Test + public void testCompareTo() { + HServerAddress hsa1 = new HServerAddress("localhost:1234"); + HServerInfo hsi1 = new HServerInfo(hsa1, 1L, 5678); + HServerAddress hsa2 = new HServerAddress("localhost:1235"); + HServerInfo hsi2 = new HServerInfo(hsa2, 1L, 5678); + assertTrue(hsi1.compareTo(hsi1) == 0); + assertTrue(hsi2.compareTo(hsi2) == 0); + int compare1 = hsi1.compareTo(hsi2); + int compare2 = hsi2.compareTo(hsi1); + assertTrue((compare1 > 0)? compare2 < 0: compare2 > 0); + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/TestHServerLoad.java b/src/test/java/org/apache/hadoop/hbase/TestHServerLoad.java new file mode 100644 index 0000000..021d663 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestHServerLoad.java @@ -0,0 +1,38 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.*; + +import org.junit.Test; + +public class TestHServerLoad { + + @Test + public void testHashCode() { + HServerLoad hsl1 = new HServerLoad(1, 1, 1); + HServerLoad hsl2 = new HServerLoad(1, 1, 1); + assertEquals(hsl1, hsl2); + assertTrue(hsl1.equals(hsl2)); + HServerLoad hsl3 = new HServerLoad(2, 2, 2); + assertNotSame(hsl1, hsl3); + assertFalse(hsl1.equals(hsl3)); + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/TestSerialization.java b/src/test/java/org/apache/hadoop/hbase/TestSerialization.java index befcdaf..56f1c6e 100644 --- a/src/test/java/org/apache/hadoop/hbase/TestSerialization.java +++ b/src/test/java/org/apache/hadoop/hbase/TestSerialization.java @@ -84,21 +84,6 @@ public class TestSerialization { assertTrue(Bytes.equals("value".getBytes(), hmw.get("key".getBytes()))); } - @Test public void testHMsg() throws Exception { - final String name = "testHMsg"; - HMsg m = new HMsg(HMsg.Type.STOP_REGIONSERVER); - byte [] mb = Writables.getBytes(m); - HMsg deserializedHMsg = (HMsg)Writables.getWritable(mb, new HMsg()); - assertTrue(m.equals(deserializedHMsg)); - m = new HMsg(HMsg.Type.STOP_REGIONSERVER, - new HRegionInfo(new HTableDescriptor(name), - HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY), - "Some message".getBytes()); - mb = Writables.getBytes(m); - deserializedHMsg = (HMsg)Writables.getWritable(mb, new HMsg()); - assertTrue(m.equals(deserializedHMsg)); - } - @Test public void testTableDescriptor() throws Exception { final String name = "testTableDescriptor"; HTableDescriptor htd = createTableDescriptor(name); @@ -129,19 +114,6 @@ public class TestSerialization { deserializedHri.getTableDesc().getFamilies().size()); } - /** - * Test ServerInfo serialization - * @throws Exception - */ - @Test public void testServerInfo() throws Exception { - HServerInfo hsi = new HServerInfo(new HServerAddress("0.0.0.0:123"), -1, - 1245, "default name"); - byte [] b = Writables.getBytes(hsi); - HServerInfo deserializedHsi = - (HServerInfo)Writables.getWritable(b, new HServerInfo()); - assertTrue(hsi.equals(deserializedHsi)); - } - @Test public void testPut() throws Exception{ byte[] row = "row".getBytes(); byte[] fam = "fam".getBytes(); diff --git a/src/test/java/org/apache/hadoop/hbase/TestServerName.java b/src/test/java/org/apache/hadoop/hbase/TestServerName.java new file mode 100644 index 0000000..ba94c10 --- /dev/null +++ b/src/test/java/org/apache/hadoop/hbase/TestServerName.java @@ -0,0 +1,62 @@ +/** + * Copyright 2011 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.*; + +import java.util.HashSet; +import java.util.Set; + +import org.junit.Test; + +public class TestServerName { + @Test + public void testServerName() { + ServerName sn = new ServerName("www.example.org", 1234, 5678); + assertEquals(sn.toString(), + ServerName.getServerName("www.example.org", 1234, 5678)); + assertEquals(sn.toString(), + ServerName.getServerName("www.example.org:1234", 5678)); + assertEquals(sn.toString(), + "www.example.org" + ServerName.SERVERNAME_SEPARATOR + + "1234" + ServerName.SERVERNAME_SEPARATOR + "5678"); + } + + @Test + public void getServerStartcodeFromServerName() { + ServerName sn = new ServerName("www.example.org", 1234, 5678); + assertEquals(5678, + ServerName.getServerStartcodeFromServerName(sn.toString())); + assertNotSame(5677, + ServerName.getServerStartcodeFromServerName(sn.toString())); + } + + @Test + public void testIsServer() { + ServerName sn = new ServerName("www.example.org", 1234, 5678); + ServerName sn80 = new ServerName("www.example.org", 80, 5678); + Set servers = new HashSet(); + servers.add(sn.toString()); + assertTrue(ServerName.isServer(servers, sn.toString(), false)); + assertFalse(ServerName.isServer(servers, sn80.toString(), false)); + assertTrue(ServerName.isServer(servers, "www.example.org:1234", true)); + assertFalse(ServerName.isServer(servers, "www.example.org:80", true)); + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/master/BROKE_FIX_TestKillingServersFromMaster.java b/src/test/java/org/apache/hadoop/hbase/master/BROKE_FIX_TestKillingServersFromMaster.java deleted file mode 100644 index 21b76fa..0000000 --- a/src/test/java/org/apache/hadoop/hbase/master/BROKE_FIX_TestKillingServersFromMaster.java +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Copyright 2010 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master; - -import static org.junit.Assert.assertEquals; - -import java.io.IOException; - -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HServerAddress; -import org.apache.hadoop.hbase.HServerInfo; -import org.apache.hadoop.hbase.MiniHBaseCluster; -import org.apache.hadoop.hbase.YouAreDeadException; -import org.apache.hadoop.hbase.MiniHBaseCluster.MiniHBaseClusterRegionServer; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; - -public class BROKE_FIX_TestKillingServersFromMaster { - private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private static MiniHBaseCluster cluster; - - @BeforeClass - public static void beforeAllTests() throws Exception { - TEST_UTIL.startMiniCluster(2); - cluster = TEST_UTIL.getHBaseCluster(); - } - - @AfterClass - public static void afterAllTests() throws IOException { - TEST_UTIL.shutdownMiniCluster(); - } - - @Before - public void setup() throws IOException { - TEST_UTIL.ensureSomeRegionServersAvailable(2); - } - - /** - * Test that a region server that reports with the wrong start code - * gets shut down - * See HBASE-2613 - * @throws Exception - */ - @Ignore @Test (timeout=180000) - public void testRsReportsWrongStartCode() throws Exception { - MiniHBaseClusterRegionServer firstServer = - (MiniHBaseClusterRegionServer)cluster.getRegionServer(0); - HServerInfo hsi = firstServer.getServerInfo(); - // This constructor creates a new startcode - firstServer.setHServerInfo(new HServerInfo(hsi.getServerAddress(), - hsi.getInfoPort(), hsi.getHostname())); - cluster.waitOnRegionServer(0); - assertEquals(1, cluster.getLiveRegionServerThreads().size()); - } - - /** - * Test that a region server that reports with the wrong address - * gets shut down - * See HBASE-2613 - * @throws Exception - */ - @Ignore @Test (timeout=180000) - public void testRsReportsWrongAddress() throws Exception { - MiniHBaseClusterRegionServer firstServer = - (MiniHBaseClusterRegionServer)cluster.getRegionServer(0); - firstServer.getHServerInfo().setServerAddress( - new HServerAddress("0.0.0.0", 60010)); - cluster.waitOnRegionServer(0); - assertEquals(1, cluster.getLiveRegionServerThreads().size()); - } - - /** - * Send a YouAreDeadException to the region server and expect it to shut down - * See HBASE-2691 - * @throws Exception - */ - @Ignore @Test (timeout=180000) - public void testSendYouAreDead() throws Exception { - cluster.addExceptionToSendRegionServer(0, new YouAreDeadException("bam!")); - cluster.waitOnRegionServer(0); - assertEquals(1, cluster.getLiveRegionServerThreads().size()); - } -} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/master/OOMEHMaster.java b/src/test/java/org/apache/hadoop/hbase/master/OOMEHMaster.java deleted file mode 100644 index bf5ed03..0000000 --- a/src/test/java/org/apache/hadoop/hbase/master/OOMEHMaster.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright 2007 The Apache Software Foundation - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HMsg; -import org.apache.hadoop.hbase.HRegionInfo; -import org.apache.hadoop.hbase.HServerInfo; -import org.apache.zookeeper.KeeperException; - -/** - * An HMaster that runs out of memory. - * Everytime a region server reports in, add to the retained heap of memory. - * Needs to be started manually as in - * ${HBASE_HOME}/bin/hbase ./bin/hbase org.apache.hadoop.hbase.OOMEHMaster start/code>. - */ -public class OOMEHMaster extends HMaster { - private List retainer = new ArrayList(); - - public OOMEHMaster(HBaseConfiguration conf) - throws IOException, KeeperException, InterruptedException { - super(conf); - } - - @Override - public HMsg[] regionServerReport(HServerInfo serverInfo, HMsg[] msgs, - HRegionInfo[] mostLoadedRegions) - throws IOException { - // Retain 1M. - this.retainer.add(new byte [1024 * 1024]); - return super.regionServerReport(serverInfo, msgs, mostLoadedRegions); - } - - public static void main(String[] args) throws Exception { - new HMasterCommandLine(OOMEHMaster.class).doMain(args); - } -}