Index: src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java =================================================================== --- src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java (revision 581746) +++ src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java (working copy) @@ -85,6 +85,7 @@ /** * Starts a MiniHBaseCluster on top of an existing HDFSCluster * + *
****************************************************************************
* * * * * * N O T E * * * * *
*
@@ -93,6 +94,7 @@
*
* * * * * * N O T E * * * * *
****************************************************************************
+ *
*
* @param conf
* @param nRegionNodes
@@ -287,6 +289,13 @@
}
/**
+ * @return the thread running the HMaster
+ */
+ public MasterThread getMasterThread() {
+ return this.masterThread;
+ }
+
+ /**
* Cause a region server to exit without cleaning up
*
* @param serverNumber
Index: src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestDFSAbort.java
===================================================================
--- src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestDFSAbort.java (revision 581746)
+++ src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestDFSAbort.java (working copy)
@@ -22,14 +22,27 @@
import junit.framework.TestSuite;
import junit.textui.TestRunner;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
+import java.io.PrintWriter;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
/**
* Test ability of HBase to handle DFS failure
*/
public class TestDFSAbort extends HBaseClusterTestCase {
+ private static final Log LOG =
+ LogFactory.getLog(TestDFSAbort.class.getName());
+ /** constructor */
+ public TestDFSAbort() {
+ super();
+
+ // For less frequently updated regions flush after every 2 flushes
+ conf.setInt("hbase.hregion.memcache.optionalflushcount", 2);
+ }
+
/** {@inheritDoc} */
@Override
public void setUp() throws Exception {
@@ -54,13 +67,43 @@
// created a table. Now let's yank the rug out from HBase
cluster.getDFSCluster().shutdown();
// Now wait for Mini HBase Cluster to shut down
- cluster.join();
+// cluster.join();
+ join();
} catch (Exception e) {
e.printStackTrace();
throw e;
}
}
+ private void join() {
+ if (this.cluster.regionThreads != null) {
+ synchronized(this.cluster.regionThreads) {
+ for(Thread t: this.cluster.regionThreads) {
+ join(t);
+ }
+ }
+ }
+ join(this.cluster.getMasterThread());
+ }
+
+ private void join(final Thread t) {
+ if (t == null) {
+ return;
+ }
+ for (int i = 0; t.isAlive(); i++) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ LOG.info("Continuing...", e);
+ }
+ if (i != 0 && i % 30 == 0) {
+ ReflectionUtils.printThreadInfo(new PrintWriter(System.out),
+ "Automatic Stack Trace every 30 seconds waiting on " +
+ t.getName());
+ }
+ }
+ }
+
/**
* @param args unused
*/
Index: src/contrib/hbase/src/test/org/apache/hadoop/hbase/MultiRegionTable.java
===================================================================
--- src/contrib/hbase/src/test/org/apache/hadoop/hbase/MultiRegionTable.java (revision 581746)
+++ src/contrib/hbase/src/test/org/apache/hadoop/hbase/MultiRegionTable.java (working copy)
@@ -55,6 +55,10 @@
MiniHBaseCluster cluster, FileSystem localFs, String tableName,
String columnName) throws IOException {
+ final int retries = 10;
+ final long waitTime =
+ conf.getLong("hbase.master.meta.thread.rescanfrequency", 10L * 1000L);
+
// This size should make it so we always split using the addContent
// below. After adding all data, the first region is 1.3M. Should
// set max filesize to be <= 1M.
@@ -62,7 +66,6 @@
assertTrue(conf.getLong("hbase.hregion.max.filesize",
HConstants.DEFAULT_MAX_FILE_SIZE) <= 1024 * 1024);
- final int retries = 10;
FileSystem fs = (cluster.getDFSCluster() == null) ?
localFs : cluster.getDFSCluster().getFileSystem();
assertNotNull(fs);
@@ -89,18 +92,18 @@
// Now, wait until split makes it into the meta table.
- for (int i = 0;
- i < retries && (count(meta, HConstants.COLUMN_FAMILY_STR) <= count);
- i++) {
-
+ int oldCount = count;
+ for (int i = 0; i < retries; i++) {
+ count = count(meta, HConstants.COLUMN_FAMILY_STR);
+ if (count > oldCount) {
+ break;
+ }
try {
- Thread.sleep(5000);
+ Thread.sleep(waitTime);
} catch (InterruptedException e) {
// continue
}
}
- int oldCount = count;
- count = count(meta, HConstants.COLUMN_FAMILY_STR);
if (count <= oldCount) {
throw new IOException("Failed waiting on splits to show up");
}
@@ -126,7 +129,7 @@
// Recalibrate will cause us to wait on new regions' deployment
- recalibrate(t, new Text(columnName), retries);
+ recalibrate(t, new Text(columnName), retries, waitTime);
// Compact a region at a time so we can test case where one region has
// no references but the other still has some
@@ -138,7 +141,7 @@
while (getSplitParentInfo(meta, parent).size() == 3) {
try {
- Thread.sleep(5000);
+ Thread.sleep(waitTime);
} catch (InterruptedException e) {
// continue
}
@@ -153,12 +156,13 @@
// Now wait until parent disappears.
LOG.info("Waiting on parent " + parent.getRegionName() + " to disappear");
- for (int i = 0;
- i < retries && getSplitParentInfo(meta, parent) != null;
- i++) {
+ for (int i = 0; i < retries; i++) {
+ if (getSplitParentInfo(meta, parent) == null) {
+ break;
+ }
try {
- Thread.sleep(5000);
+ Thread.sleep(waitTime);
} catch (InterruptedException e) {
// continue
}
@@ -167,9 +171,12 @@
// Assert cleaned up.
- for (int i = 0; i < retries && fs.exists(parentDir); i++) {
+ for (int i = 0; i < retries; i++) {
+ if (!fs.exists(parentDir)) {
+ break;
+ }
try {
- Thread.sleep(5000);
+ Thread.sleep(waitTime);
} catch (InterruptedException e) {
// continue
}
@@ -243,7 +250,7 @@
* @param retries
*/
private static void recalibrate(final HTable t, final Text column,
- final int retries) throws IOException {
+ final int retries, final long waitTime) throws IOException {
for (int i = 0; i < retries; i++) {
try {
@@ -260,7 +267,7 @@
} catch (NotServingRegionException x) {
System.out.println("it's alright");
try {
- Thread.sleep(5000);
+ Thread.sleep(waitTime);
} catch (InterruptedException e) {
// continue
}
Index: src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestLogRolling.java
===================================================================
--- src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestLogRolling.java (revision 581746)
+++ src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestLogRolling.java (working copy)
@@ -68,6 +68,10 @@
// Increase the amount of time between client retries
conf.setLong("hbase.client.pause", 15 * 1000);
+ // Reduce thread wake frequency so that other threads can get
+ // a chance to run.
+ conf.setInt(HConstants.THREAD_WAKE_FREQUENCY, 2 * 1000);
+
String className = this.getClass().getName();
StringBuilder v = new StringBuilder(className);
while (v.length() < 1000) {
Index: src/contrib/hbase/src/test/org/apache/hadoop/hbase/StaticTestEnvironment.java
===================================================================
--- src/contrib/hbase/src/test/org/apache/hadoop/hbase/StaticTestEnvironment.java (revision 581746)
+++ src/contrib/hbase/src/test/org/apache/hadoop/hbase/StaticTestEnvironment.java (working copy)
@@ -86,25 +86,25 @@
} else if(value.equalsIgnoreCase("WARN")) {
logLevel = Level.WARN;
}
+ }
- ConsoleAppender consoleAppender = null;
- for(Enumeration