Index: src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java (revision 0) +++ src/test/java/org/apache/hadoop/hbase/master/TestClockSkewDetection.java (revision 0) @@ -0,0 +1,94 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.UnknownRegionException; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.conf.Configuration; + +import java.io.IOException; +import junit.framework.Assert; +import org.apache.hadoop.hbase.ClockOutOfSyncException; +import org.apache.hadoop.hbase.HServerAddress; +import org.apache.hadoop.hbase.HServerInfo; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + + + +public class TestClockSkewDetection { + private static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + private static final Log LOG = + LogFactory.getLog(TestClockSkewDetection.class); + private static final byte[] TABLENAME = Bytes.toBytes("TestMaster"); + private static final byte[] FAMILYNAME = Bytes.toBytes("fam"); + + @BeforeClass + public static void beforeAllTests() throws Exception { + // Start a cluster of 1 regionservers. + TEST_UTIL.startMiniCluster(1); + } + + @AfterClass + public static void afterAllTests() throws IOException { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testClockSkewDetection() throws Exception { + //ServerManager sm = new ServerManager(new HMaster(new Configuration()), null, + // null); + + HMaster m = TEST_UTIL.getHBaseCluster().getMaster(); + ServerManager sm = m.getServerManager(); + + //there should be no exception + LOG.debug("regionServerStartup 1"); + HServerInfo hsi1 = new HServerInfo(new HServerAddress("example.org:1234"), + System.currentTimeMillis(), -1, "example.com"); + sm.regionServerStartup(hsi1, System.currentTimeMillis()); + sm.expireServer(hsi1); + + Configuration c = TEST_UTIL.getConfiguration(); + long maxSkew = c.getInt("hbase.master.regionserver.maxClockSkewMS", 30000); + + try { + LOG.debug("regionServerStartup 2"); + HServerInfo hsi2 = new HServerInfo(new HServerAddress("example.org:1235"), + System.currentTimeMillis(), -1, "example.com"); + sm.regionServerStartup(hsi2, System.currentTimeMillis() - maxSkew * 2); + sm.expireServer(hsi2); + Assert.assertTrue("HMaster should have thrown an ClockOutOfSyncException " + + "but didn't.", false); + } catch(ClockOutOfSyncException e) { + //we want an exception + LOG.info("Recieved expected exception: "+e); + } + } +} \ No newline at end of file Index: src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java (revision 1032791) +++ src/main/java/org/apache/hadoop/hbase/ipc/HMasterRegionInterface.java (working copy) @@ -40,11 +40,13 @@ /** * Called when a region server first starts * @param info server info + * @param serverCurrentTime The current time of the region server in ms * @throws IOException e * @return Configuration for the regionserver to use: e.g. filesystem, * hbase rootdir, etc. */ - public MapWritable regionServerStartup(HServerInfo info) throws IOException; + public MapWritable regionServerStartup(HServerInfo info, + long serverCurrentTime) throws IOException; /** * Called to renew lease, tell master what the region server is doing and to Index: src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java (revision 1032791) +++ src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java (working copy) @@ -78,7 +78,8 @@ *
  • Version 24: HBASE-2473, create table with regions.
  • *
  • Version 25: Added openRegion and Stoppable/Abortable to API.
  • *
  • Version 26: New master and Increment, 0.90 version bump.
  • + *
  • Version 27: HBASE-3168, Added serverCurrentTime to regionServerStartup in HMasterRegionInterface.
  • * */ - public static final long versionID = 26L; + public static final long versionID = 27L; } Index: src/main/java/org/apache/hadoop/hbase/master/HMaster.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/HMaster.java (revision 1032791) +++ src/main/java/org/apache/hadoop/hbase/master/HMaster.java (working copy) @@ -577,7 +577,10 @@ } } - public MapWritable regionServerStartup(final HServerInfo serverInfo) + /** @see HMasterRegionInterface#regionServerStartup + */ + public MapWritable regionServerStartup(final HServerInfo serverInfo, + final long serverCurrentTime) throws IOException { // Set the ip into the passed in serverInfo. Its ip is more than likely // not the ip that the master sees here. See at end of this method where @@ -589,7 +592,7 @@ serverInfo.setServerAddress(new HServerAddress(rsAddress, serverInfo.getServerAddress().getPort())); // Register with server manager - this.serverManager.regionServerStartup(serverInfo); + this.serverManager.regionServerStartup(serverInfo, serverCurrentTime); // Send back some config info MapWritable mw = createConfigurationSubset(); mw.put(new Text("hbase.regionserver.address"), new Text(rsAddress)); Index: src/main/java/org/apache/hadoop/hbase/master/ServerManager.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (revision 1032791) +++ src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (working copy) @@ -31,6 +31,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ClockOutOfSyncException; import org.apache.hadoop.hbase.HMsg; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HServerAddress; @@ -116,9 +117,10 @@ /** * Let the server manager know a new regionserver has come online * @param serverInfo + * @param serverCurrentTime The current time of the region server in ms * @throws IOException */ - void regionServerStartup(final HServerInfo serverInfo) + void regionServerStartup(final HServerInfo serverInfo, long serverCurrentTime) throws IOException { // Test for case where we get a region startup message from a regionserver // that has been quickly restarted but whose znode expiration handler has @@ -130,6 +132,7 @@ HServerInfo info = new HServerInfo(serverInfo); checkIsDead(info.getServerName(), "STARTUP"); checkAlreadySameHostPort(info); + checkClockSkew(info, serverCurrentTime); recordNewServer(info, false, null); } @@ -168,6 +171,28 @@ } /** + * Checks if the clock skew between the server and the master. If the clock + * skew is too much it will throw an Exception. + * @throws ClockOutOfSyncException + */ + private void checkClockSkew(final HServerInfo serverInfo, + final long serverCurrentTime) + throws ClockOutOfSyncException { + Configuration c = master.getConfiguration(); + long skew = System.currentTimeMillis() - serverCurrentTime; + long maxSkew = c.getInt("hbase.master.regionserver.maxClockSkewMS", 30000); + if (skew > maxSkew) + { + String message = "Server " + serverInfo.getServerName() + + " rejected; Server system time is " + + "too far out of sync with master. Time difference: "+ + skew+"ms > "+maxSkew+"ms"; + LOG.debug(message); + throw new ClockOutOfSyncException(message); + } + } + + /** * If this server is on the dead list, reject it with a LeaseStillHeldException * @param serverName Server name formatted as host_port_startcode. * @param what START or REPORT Index: src/main/java/org/apache/hadoop/hbase/ClockOutOfSyncException.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/ClockOutOfSyncException.java (revision 0) +++ src/main/java/org/apache/hadoop/hbase/ClockOutOfSyncException.java (revision 0) @@ -0,0 +1,33 @@ +/** + * Copyright 2010 The Apache Software Foundation + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.io.IOException; + +/** + * This exception is thrown by the master when a region server clock skew is + * too high. + */ +@SuppressWarnings("serial") +public class ClockOutOfSyncException extends IOException { + public ClockOutOfSyncException(String message) { + super(message); + } +} \ No newline at end of file Index: src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (revision 1032791) +++ src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (working copy) @@ -1415,7 +1415,8 @@ this.serverInfo.getServerAddress()); this.serverInfo.setLoad(buildServerLoad()); LOG.info("Telling master at " + masterAddress + " that we are up"); - result = this.hbaseMaster.regionServerStartup(this.serverInfo); + result = this.hbaseMaster.regionServerStartup(this.serverInfo, + System.currentTimeMillis()); break; } catch (IOException e) { LOG.warn("error telling master we are up", e);