diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java index a8847f6..9eeb5c2 100644 --- hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java +++ hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedList.java @@ -170,6 +170,8 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { private static final String GENERATOR_NUM_MAPPERS_KEY = "IntegrationTestBigLinkedList.generator.map.tasks"; + protected int NUM_SLAVES_BASE = 3; // number of slaves for the cluster + static class CINode { long key; long prev; @@ -648,7 +650,7 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { * Executes Generate and Verify in a loop. Data is not cleaned between runs, so each iteration * adds more data. */ - private static class Loop extends Configured implements Tool { + static class Loop extends Configured implements Tool { private static final Log LOG = LogFactory.getLog(Loop.class); @@ -916,12 +918,12 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { return node; } - private IntegrationTestingUtility util; + protected IntegrationTestingUtility util; @Before public void setUp() throws Exception { util = getTestingUtil(); - util.initializeCluster(3); + util.initializeCluster(this.NUM_SLAVES_BASE); this.setConf(util.getConfiguration()); } @@ -939,7 +941,7 @@ public class IntegrationTestBigLinkedList extends Configured implements Tool { org.junit.Assert.assertEquals(0, ret); } - private IntegrationTestingUtility getTestingUtil() { + protected IntegrationTestingUtility getTestingUtil() { if (this.util == null) { if (getConf() == null) { this.util = new IntegrationTestingUtility(); diff --git hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithChaosMonkey.java hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithChaosMonkey.java new file mode 100644 index 0000000..f6b26a5 --- /dev/null +++ hbase-it/src/test/java/org/apache/hadoop/hbase/test/IntegrationTestBigLinkedListWithChaosMonkey.java @@ -0,0 +1,72 @@ +package org.apache.hadoop.hbase.test; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.IntegrationTestingUtility; +import org.apache.hadoop.hbase.IntegrationTests; +import org.apache.hadoop.hbase.util.ChaosMonkey; +import org.apache.hadoop.util.ToolRunner; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + + +/** + * This is the same integration test as {@link IntegrationTestBigLinkedList} while killing the + * region servers and the master(s) randomly + */ +@Category(IntegrationTests.class) +public class IntegrationTestBigLinkedListWithChaosMonkey extends IntegrationTestBigLinkedList { + private static final Log LOG = LogFactory + .getLog(IntegrationTestBigLinkedListWithChaosMonkey.class); + + private ChaosMonkey monkey; + + @Before + public void setUp() throws Exception { + this.NUM_SLAVES_BASE = 5; + super.setUp(); + monkey = new ChaosMonkey(util, ChaosMonkey.EVERY_MINUTE_RANDOM_ACTION_POLICY); + LOG.info("Chaos Monkey Starting"); + monkey.start(); + } + + @After + public void tearDown() throws Exception { + if (monkey != null) { + monkey.stop("test has finished, that's why"); + monkey.waitForStop(); + } + super.tearDown(); + } + + @Test + public void testContinuousIngest() throws IOException, Exception { + // Loop + int ret = ToolRunner.run( + getTestingUtil().getConfiguration(), + new Loop(), + new String[] { "1", "1", "1000000", + util.getDataTestDirOnTestFS("IntegrationTestBigLinkedListWithChaosMonkey").toString(), + "1" }); + org.junit.Assert.assertEquals(0, ret); + } + + public static void main(String[] args) throws Exception { + + IntegrationTestBigLinkedListWithChaosMonkey test = + new IntegrationTestBigLinkedListWithChaosMonkey(); + test.setUp(); + + // run the test + int ret = ToolRunner.run(test.getConf(), new IntegrationTestBigLinkedListWithChaosMonkey(), + args); + + test.tearDown(); + System.exit(ret); + } +} diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java index dd3dcb1..51ae352 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java @@ -290,12 +290,20 @@ public class RpcServer implements RpcServerInterface { @Override public String toString() { - String serviceName = this.connection.service != null? - this.connection.service.getDescriptorForType().getName(): "null"; - return "callId: " + this.id + " service: " + serviceName + " methodName: " + - ((this.md != null)? this.md.getName(): null) + " param: " + - (this.param != null? IPCUtil.getRequestShortTextFormat(this.param): "") + - " connection: " + connection.toString(); + return toShortString() + " param: " + + (this.param != null ? IPCUtil.getRequestShortTextFormat(this.param) : ""); + } + + /* + * Short string representation without param info because param itself could huge depends on the + * payload of a command + */ + String toShortString() { + String serviceName = this.connection.service != null ? this.connection.service + .getDescriptorForType().getName() : "null"; + return "callId: " + this.id + " service: " + serviceName + " methodName: " + + ((this.md != null) ? this.md.getName() : null) + " connection: " + + connection.toString(); } protected synchronized void setSaslTokenResponse(ByteBuffer response) { @@ -986,7 +994,8 @@ public class RpcServer implements RpcServerInterface { } } finally { if (error && call != null) { - LOG.warn(getName() + call.toString() + ": output error"); + LOG.warn(getName() + ((call.size > 512) ? call.toShortString() : call.toString()) + + ": output error"); done = true; // error. no more data for this channel. closeConnection(call.connection); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 5267597..9821f6d 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -914,7 +914,7 @@ public class HRegionServer implements ClientProtos.ClientService.BlockingInterfa } //fsOk flag may be changed when closing regions throws exception. - if (!this.killed && this.fsOk) { + if (this.fsOk) { closeWAL(!abortRequested); }