Index: src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java
===================================================================
--- src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java (revision 0)
+++ src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java (revision 0)
@@ -0,0 +1,87 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.conf.Configuration;
+
+import java.io.IOException;
+import junit.framework.Assert;
+import org.apache.hadoop.hbase.ClockOutOfSyncException;
+import org.apache.hadoop.hbase.HServerAddress;
+import org.apache.hadoop.hbase.HServerInfo;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+
+
+public class TestClockSkewDetection {
+ private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ private static final Log LOG = LogFactory.getLog(TestClockSkewDetection.class);
+ private static final byte[] TABLENAME = Bytes.toBytes("TestMaster");
+ private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
+
+ @BeforeClass
+ public static void beforeAllTests() throws Exception {
+ // Start a cluster of 1 regionservers.
+ TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void afterAllTests() throws IOException {
+ TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testClockSkewDetection() throws Exception {
+ MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+ HMaster m = cluster.getMaster();
+
+ ServerManager sm = m.getServerManager();
+
+ //there should be no exception
+ LOG.debug("regionServerStartup 1");
+ HServerInfo hsi1 = new HServerInfo(new HServerAddress("example.org:1234"),
+ System.currentTimeMillis(), -1, "example.com");
+ sm.regionServerStartup(hsi1, System.currentTimeMillis());
+ sm.expireServer(hsi1);
+
+ Configuration c = TEST_UTIL.getConfiguration();
+ long maxSkew = c.getInt("hbase.master.regionserver.maxClockSkewMS", 30000);
+
+ try {
+ LOG.debug("regionServerStartup 2");
+ HServerInfo hsi2 = new HServerInfo(new HServerAddress("example.org:1235"),
+ System.currentTimeMillis(), -1, "example.com");
+ sm.regionServerStartup(hsi2, System.currentTimeMillis() - maxSkew * 2);
+ sm.expireServer(hsi2);
+ Assert.assertTrue("HMaster should have thrown an ClockOutOfSyncException but didn't.", false);
+ } catch(ClockOutOfSyncException e) {
+ //we want an exception
+ LOG.info("Recieved expected exception: "+e);
+ }
+ }
+}
\ No newline at end of file
Index: src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java
===================================================================
--- src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java (revision 1032652)
+++ src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java (working copy)
@@ -40,11 +40,12 @@
/**
* Called when a region server first starts
* @param info server info
+ * @param serverCurrentTime The current time of the region server in ms
* @throws IOException e
* @return Configuration for the regionserver to use: e.g. filesystem,
* hbase rootdir, etc.
*/
- public MapWritable regionServerStartup(HServerInfo info) throws IOException;
+ public MapWritable regionServerStartup(HServerInfo info, long serverCurrentTime) throws IOException;
/**
* Called to renew lease, tell master what the region server is doing and to
Index: src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java
===================================================================
--- src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java (revision 1032652)
+++ src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java (working copy)
@@ -78,7 +78,8 @@
*
Version 24: HBASE-2473, create table with regions.
* Version 25: Added openRegion and Stoppable/Abortable to API.
* Version 26: New master and Increment, 0.90 version bump.
+ * Version 27: HBASE-3168, Added serverCurrentTime to regionServerStartup in HMasterRegionInterface.
*
*/
- public static final long versionID = 26L;
+ public static final long versionID = 27L;
}
Index: src/main/java/org/apache/hadoop/hbase/master/HMaster.java
===================================================================
--- src/main/java/org/apache/hadoop/hbase/master/HMaster.java (revision 1032652)
+++ src/main/java/org/apache/hadoop/hbase/master/HMaster.java (working copy)
@@ -577,7 +577,9 @@
}
}
- public MapWritable regionServerStartup(final HServerInfo serverInfo)
+ /** @see HMasterRegionInterface#regionServerStartup
+ */
+ public MapWritable regionServerStartup(final HServerInfo serverInfo, final long serverCurrentTime)
throws IOException {
// Set the ip into the passed in serverInfo. Its ip is more than likely
// not the ip that the master sees here. See at end of this method where
@@ -589,7 +591,7 @@
serverInfo.setServerAddress(new HServerAddress(rsAddress,
serverInfo.getServerAddress().getPort()));
// Register with server manager
- this.serverManager.regionServerStartup(serverInfo);
+ this.serverManager.regionServerStartup(serverInfo, serverCurrentTime);
// Send back some config info
MapWritable mw = createConfigurationSubset();
mw.put(new Text("hbase.regionserver.address"), new Text(rsAddress));
Index: src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
===================================================================
--- src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (revision 1032652)
+++ src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (working copy)
@@ -31,6 +31,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ClockOutOfSyncException;
import org.apache.hadoop.hbase.HMsg;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerAddress;
@@ -116,9 +117,10 @@
/**
* Let the server manager know a new regionserver has come online
* @param serverInfo
+ * @param serverCurrentTime The current time of the region server in ms
* @throws IOException
*/
- void regionServerStartup(final HServerInfo serverInfo)
+ void regionServerStartup(final HServerInfo serverInfo, long serverCurrentTime)
throws IOException {
// Test for case where we get a region startup message from a regionserver
// that has been quickly restarted but whose znode expiration handler has
@@ -130,6 +132,7 @@
HServerInfo info = new HServerInfo(serverInfo);
checkIsDead(info.getServerName(), "STARTUP");
checkAlreadySameHostPort(info);
+ checkClockSkew(info, serverCurrentTime);
recordNewServer(info, false, null);
}
@@ -168,6 +171,25 @@
}
/**
+ * Checks if the clock skew between the server and the master. If the clock
+ * skew is too much it will throw an Exception.
+ * @throws ClockOutOfSyncException
+ */
+ private void checkClockSkew(final HServerInfo serverInfo, final long serverCurrentTime)
+ throws ClockOutOfSyncException {
+ Configuration c = master.getConfiguration();
+ long skew = System.currentTimeMillis() - serverCurrentTime;
+ long maxSkew = c.getInt("hbase.master.regionserver.maxClockSkewMS", 30000);
+ if (skew > maxSkew)
+ {
+ String message = "Server " + serverInfo.getServerName() + " rejected; Server system time is " +
+ "too far out of sync with master. Time difference: "+skew+"ms > "+maxSkew+"ms";
+ LOG.debug(message);
+ throw new ClockOutOfSyncException(message);
+ }
+ }
+
+ /**
* If this server is on the dead list, reject it with a LeaseStillHeldException
* @param serverName Server name formatted as host_port_startcode.
* @param what START or REPORT
Index: src/main/java/org/apache/hadoop/hbase/ClockOutOfSyncException.java
===================================================================
--- src/main/java/org/apache/hadoop/hbase/ClockOutOfSyncException.java (revision 0)
+++ src/main/java/org/apache/hadoop/hbase/ClockOutOfSyncException.java (revision 0)
@@ -0,0 +1,33 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+/**
+ * This exception is thrown by the master when a region server clock skew is
+ * too high.
+ */
+@SuppressWarnings("serial")
+public class ClockOutOfSyncException extends IOException {
+ public ClockOutOfSyncException(String message) {
+ super(message);
+ }
+}
\ No newline at end of file
Index: src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
===================================================================
--- src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (revision 1032652)
+++ src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (working copy)
@@ -1415,7 +1415,7 @@
this.serverInfo.getServerAddress());
this.serverInfo.setLoad(buildServerLoad());
LOG.info("Telling master at " + masterAddress + " that we are up");
- result = this.hbaseMaster.regionServerStartup(this.serverInfo);
+ result = this.hbaseMaster.regionServerStartup(this.serverInfo, System.currentTimeMillis());
break;
} catch (IOException e) {
LOG.warn("error telling master we are up", e);