Index: hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java (revision 1503919) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java (working copy) @@ -88,6 +88,7 @@ conf1.setBoolean("dfs.support.append", true); conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100); conf1.setInt("replication.stats.thread.period.seconds", 5); + conf1.setBoolean("hbase.tests.use.shortcircuit.reads", false); utility1 = new HBaseTestingUtility(conf1); utility1.startMiniZKCluster(); @@ -105,6 +106,7 @@ conf2.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6); conf2.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true); conf2.setBoolean("dfs.support.append", true); + conf2.setBoolean("hbase.tests.use.shortcircuit.reads", false); utility2 = new HBaseTestingUtility(conf2); utility2.setZkCluster(miniZK); @@ -127,7 +129,7 @@ HBaseAdmin admin1 = new HBaseAdmin(conf1); HBaseAdmin admin2 = new HBaseAdmin(conf2); admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); - admin2.createTable(table); + admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); htable1 = new HTable(conf1, tableName); htable1.setWriteBufferSize(1024); htable2 = new HTable(conf2, tableName); Index: hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailover.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailover.java (revision 1503919) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailover.java (working copy) @@ -1,133 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.replication; - - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.LargeTests; -import org.apache.hadoop.hbase.exceptions.UnknownScannerException; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import static org.junit.Assert.fail; - -@Category(LargeTests.class) -public class TestReplicationQueueFailover extends TestReplicationBase { - - private static final Log LOG = LogFactory.getLog(TestReplicationQueueFailover.class); - - /** - * Load up multiple tables over 2 region servers and kill a source during - * the upload. The failover happens internally. - * - * WARNING this test sometimes fails because of HBASE-3515 - * - * @throws Exception - */ - @Test(timeout=300000) - public void queueFailover() throws Exception { - // killing the RS with .META. can result into failed puts until we solve - // IO fencing - int rsToKill1 = - utility1.getHBaseCluster().getServerWithMeta() == 0 ? 1 : 0; - int rsToKill2 = - utility2.getHBaseCluster().getServerWithMeta() == 0 ? 1 : 0; - - // Takes about 20 secs to run the full loading, kill around the middle - Thread killer1 = killARegionServer(utility1, 7500, rsToKill1); - Thread killer2 = killARegionServer(utility2, 10000, rsToKill2); - - LOG.info("Start loading table"); - int initialCount = utility1.loadTable(htable1, famName); - LOG.info("Done loading table"); - killer1.join(5000); - killer2.join(5000); - LOG.info("Done waiting for threads"); - - Result[] res; - while (true) { - try { - Scan scan = new Scan(); - ResultScanner scanner = htable1.getScanner(scan); - res = scanner.next(initialCount); - scanner.close(); - break; - } catch (UnknownScannerException ex) { - LOG.info("Cluster wasn't ready yet, restarting scanner"); - } - } - // Test we actually have all the rows, we may miss some because we - // don't have IO fencing. - if (res.length != initialCount) { - LOG.warn("We lost some rows on the master cluster!"); - // We don't really expect the other cluster to have more rows - initialCount = res.length; - } - - int lastCount = 0; - - final long start = System.currentTimeMillis(); - int i = 0; - while (true) { - if (i==NB_RETRIES-1) { - fail("Waited too much time for queueFailover replication. " + - "Waited "+(System.currentTimeMillis() - start)+"ms."); - } - Scan scan2 = new Scan(); - ResultScanner scanner2 = htable2.getScanner(scan2); - Result[] res2 = scanner2.next(initialCount * 2); - scanner2.close(); - if (res2.length < initialCount) { - if (lastCount < res2.length) { - i--; // Don't increment timeout if we make progress - } else { - i++; - } - lastCount = res2.length; - LOG.info("Only got " + lastCount + " rows instead of " + - initialCount + " current i=" + i); - Thread.sleep(SLEEP_TIME*2); - } else { - break; - } - } - } - - private static Thread killARegionServer(final HBaseTestingUtility utility, - final long timeout, final int rs) { - Thread killer = new Thread() { - public void run() { - try { - Thread.sleep(timeout); - utility.expireRegionServerSession(rs); - } catch (Exception e) { - LOG.error("Couldn't kill a region server", e); - } - } - }; - killer.setDaemon(true); - killer.start(); - return killer; - } -} Index: hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillMasterRS.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillMasterRS.java (revision 0) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillMasterRS.java (revision 0) @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import org.apache.hadoop.hbase.LargeTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Runs the TestReplicationKillRS test and selects the RS to kill in the master cluster + * Do not add other tests in this class. + */ +@Category(LargeTests.class) +public class TestReplicationKillMasterRS extends TestReplicationKillRS { + + @Test(timeout=300000) + public void killOneMasterRS() throws Exception { + loadTableAndKillRS(utility1); + } + +} Index: hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillSlaveRS.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillSlaveRS.java (revision 0) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillSlaveRS.java (revision 0) @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + +import org.apache.hadoop.hbase.LargeTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Runs the TestReplicationKillRS test and selects the RS to kill in the slave cluster + * Do not add other tests in this class. + */ +@Category(LargeTests.class) +public class TestReplicationKillSlaveRS extends TestReplicationKillRS { + + @Test(timeout=300000) + public void killOneSlaveRS() throws Exception { + loadTableAndKillRS(utility2); + } +} Index: hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillRS.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillRS.java (revision 0) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillRS.java (revision 0) @@ -0,0 +1,128 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.replication; + + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.LargeTests; +import org.apache.hadoop.hbase.exceptions.UnknownScannerException; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.fail; + +@Category(LargeTests.class) +public class TestReplicationKillRS extends TestReplicationBase { + + private static final Log LOG = LogFactory.getLog(TestReplicationKillRS.class); + + /** + * Load up 1 tables over 2 region servers and kill a source during + * the upload. The failover happens internally. + * + * WARNING this test sometimes fails because of HBASE-3515 + * + * @throws Exception + */ + public void loadTableAndKillRS(HBaseTestingUtility util) throws Exception { + // killing the RS with .META. can result into failed puts until we solve + // IO fencing + int rsToKill1 = + util.getHBaseCluster().getServerWithMeta() == 0 ? 1 : 0; + + // Takes about 20 secs to run the full loading, kill around the middle + Thread killer = killARegionServer(util, 5000, rsToKill1); + + LOG.info("Start loading table"); + int initialCount = utility1.loadTable(htable1, famName); + LOG.info("Done loading table"); + killer.join(5000); + LOG.info("Done waiting for threads"); + + Result[] res; + while (true) { + try { + Scan scan = new Scan(); + ResultScanner scanner = htable1.getScanner(scan); + res = scanner.next(initialCount); + scanner.close(); + break; + } catch (UnknownScannerException ex) { + LOG.info("Cluster wasn't ready yet, restarting scanner"); + } + } + // Test we actually have all the rows, we may miss some because we + // don't have IO fencing. + if (res.length != initialCount) { + LOG.warn("We lost some rows on the master cluster!"); + // We don't really expect the other cluster to have more rows + initialCount = res.length; + } + + int lastCount = 0; + + final long start = System.currentTimeMillis(); + int i = 0; + while (true) { + if (i==NB_RETRIES-1) { + fail("Waited too much time for queueFailover replication. " + + "Waited "+(System.currentTimeMillis() - start)+"ms."); + } + Scan scan2 = new Scan(); + ResultScanner scanner2 = htable2.getScanner(scan2); + Result[] res2 = scanner2.next(initialCount * 2); + scanner2.close(); + if (res2.length < initialCount) { + if (lastCount < res2.length) { + i--; // Don't increment timeout if we make progress + } else { + i++; + } + lastCount = res2.length; + LOG.info("Only got " + lastCount + " rows instead of " + + initialCount + " current i=" + i); + Thread.sleep(SLEEP_TIME*2); + } else { + break; + } + } + } + + private static Thread killARegionServer(final HBaseTestingUtility utility, + final long timeout, final int rs) { + Thread killer = new Thread() { + public void run() { + try { + Thread.sleep(timeout); + utility.expireRegionServerSession(rs); + } catch (Exception e) { + LOG.error("Couldn't kill a region server", e); + } + } + }; + killer.setDaemon(true); + killer.start(); + return killer; + } +} Index: hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillMasterRSCompressed.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillMasterRSCompressed.java (revision 0) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationKillMasterRSCompressed.java (revision 0) @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.replication; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.LargeTests; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +/** + * Run the same test as TestReplicationKillMasterRS but with HLog compression enabled + * Do not add other tests in this class. + */ +@Category(LargeTests.class) +public class TestReplicationKillMasterRSCompressed extends TestReplicationKillMasterRS { + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + conf1.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); + TestReplicationBase.setUpBeforeClass(); + } +} Index: hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailoverCompressed.java =================================================================== --- hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailoverCompressed.java (revision 1503919) +++ hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationQueueFailoverCompressed.java (working copy) @@ -1,40 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.replication; - -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.LargeTests; -import org.junit.BeforeClass; -import org.junit.experimental.categories.Category; - -/** - * Run the same test as TestReplication but with HLog compression enabled - */ -@Category(LargeTests.class) -public class TestReplicationQueueFailoverCompressed extends TestReplicationQueueFailover { - - /** - * @throws java.lang.Exception - */ - @BeforeClass - public static void setUpBeforeClass() throws Exception { - conf1.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true); - TestReplicationBase.setUpBeforeClass(); - } -}