Index: src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java =================================================================== --- src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java (revision 581709) +++ src/contrib/hbase/src/test/org/apache/hadoop/hbase/MiniHBaseCluster.java (working copy) @@ -85,6 +85,7 @@ /** * Starts a MiniHBaseCluster on top of an existing HDFSCluster * + *
    ****************************************************************************
    *            *  *  *  *  *  N O T E  *  *  *  *  *
    *
@@ -93,6 +94,7 @@
    *
    *            *  *  *  *  *  N O T E  *  *  *  *  *
    ****************************************************************************
+   *
* * @param conf * @param nRegionNodes @@ -287,6 +289,13 @@ } /** + * @return the thread running the HMaster + */ + public MasterThread getMasterThread() { + return this.masterThread; + } + + /** * Cause a region server to exit without cleaning up * * @param serverNumber Index: src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestDFSAbort.java =================================================================== --- src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestDFSAbort.java (revision 581709) +++ src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestDFSAbort.java (working copy) @@ -22,14 +22,27 @@ import junit.framework.TestSuite; import junit.textui.TestRunner; -import org.apache.log4j.Level; -import org.apache.log4j.Logger; +import java.io.PrintWriter; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + /** * Test ability of HBase to handle DFS failure */ public class TestDFSAbort extends HBaseClusterTestCase { + private static final Log LOG = + LogFactory.getLog(TestDFSAbort.class.getName()); + /** constructor */ + public TestDFSAbort() { + super(); + + // For less frequently updated regions flush after every 2 flushes + conf.setInt("hbase.hregion.memcache.optionalflushcount", 2); + } + /** {@inheritDoc} */ @Override public void setUp() throws Exception { @@ -54,13 +67,43 @@ // created a table. Now let's yank the rug out from HBase cluster.getDFSCluster().shutdown(); // Now wait for Mini HBase Cluster to shut down - cluster.join(); +// cluster.join(); + join(); } catch (Exception e) { e.printStackTrace(); throw e; } } + private void join() { + if (this.cluster.regionThreads != null) { + synchronized(this.cluster.regionThreads) { + for(Thread t: this.cluster.regionThreads) { + join(t); + } + } + } + join(this.cluster.getMasterThread()); + } + + private void join(final Thread t) { + if (t == null) { + return; + } + for (int i = 0; t.isAlive(); i++) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + LOG.info("Continuing...", e); + } + if (i != 0 && i % 30 == 0) { + ReflectionUtils.printThreadInfo(new PrintWriter(System.out), + "Automatic Stack Trace every 30 seconds waiting on " + + t.getName()); + } + } + } + /** * @param args unused */ Index: src/contrib/hbase/src/test/org/apache/hadoop/hbase/MultiRegionTable.java =================================================================== --- src/contrib/hbase/src/test/org/apache/hadoop/hbase/MultiRegionTable.java (revision 581709) +++ src/contrib/hbase/src/test/org/apache/hadoop/hbase/MultiRegionTable.java (working copy) @@ -55,6 +55,10 @@ MiniHBaseCluster cluster, FileSystem localFs, String tableName, String columnName) throws IOException { + final int retries = 10; + final long waitTime = + conf.getLong("hbase.master.meta.thread.rescanfrequency", 10L * 1000L); + // This size should make it so we always split using the addContent // below. After adding all data, the first region is 1.3M. Should // set max filesize to be <= 1M. @@ -62,7 +66,6 @@ assertTrue(conf.getLong("hbase.hregion.max.filesize", HConstants.DEFAULT_MAX_FILE_SIZE) <= 1024 * 1024); - final int retries = 10; FileSystem fs = (cluster.getDFSCluster() == null) ? localFs : cluster.getDFSCluster().getFileSystem(); assertNotNull(fs); @@ -89,18 +92,18 @@ // Now, wait until split makes it into the meta table. - for (int i = 0; - i < retries && (count(meta, HConstants.COLUMN_FAMILY_STR) <= count); - i++) { - + int oldCount = count; + for (int i = 0; i < retries; i++) { + count = count(meta, HConstants.COLUMN_FAMILY_STR); + if (count > oldCount) { + break; + } try { - Thread.sleep(5000); + Thread.sleep(waitTime); } catch (InterruptedException e) { // continue } } - int oldCount = count; - count = count(meta, HConstants.COLUMN_FAMILY_STR); if (count <= oldCount) { throw new IOException("Failed waiting on splits to show up"); } @@ -126,7 +129,7 @@ // Recalibrate will cause us to wait on new regions' deployment - recalibrate(t, new Text(columnName), retries); + recalibrate(t, new Text(columnName), retries, waitTime); // Compact a region at a time so we can test case where one region has // no references but the other still has some @@ -138,7 +141,7 @@ while (getSplitParentInfo(meta, parent).size() == 3) { try { - Thread.sleep(5000); + Thread.sleep(waitTime); } catch (InterruptedException e) { // continue } @@ -153,12 +156,13 @@ // Now wait until parent disappears. LOG.info("Waiting on parent " + parent.getRegionName() + " to disappear"); - for (int i = 0; - i < retries && getSplitParentInfo(meta, parent) != null; - i++) { + for (int i = 0; i < retries; i++) { + if (getSplitParentInfo(meta, parent) == null) { + break; + } try { - Thread.sleep(5000); + Thread.sleep(waitTime); } catch (InterruptedException e) { // continue } @@ -167,9 +171,12 @@ // Assert cleaned up. - for (int i = 0; i < retries && fs.exists(parentDir); i++) { + for (int i = 0; i < retries; i++) { + if (!fs.exists(parentDir)) { + break; + } try { - Thread.sleep(5000); + Thread.sleep(waitTime); } catch (InterruptedException e) { // continue } @@ -243,7 +250,7 @@ * @param retries */ private static void recalibrate(final HTable t, final Text column, - final int retries) throws IOException { + final int retries, final long waitTime) throws IOException { for (int i = 0; i < retries; i++) { try { @@ -260,7 +267,7 @@ } catch (NotServingRegionException x) { System.out.println("it's alright"); try { - Thread.sleep(5000); + Thread.sleep(waitTime); } catch (InterruptedException e) { // continue } Index: src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestLogRolling.java =================================================================== --- src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestLogRolling.java (revision 581709) +++ src/contrib/hbase/src/test/org/apache/hadoop/hbase/TestLogRolling.java (working copy) @@ -68,6 +68,10 @@ // Increase the amount of time between client retries conf.setLong("hbase.client.pause", 15 * 1000); + // Reduce thread wake frequency so that other threads can get + // a chance to run. + conf.setInt(HConstants.THREAD_WAKE_FREQUENCY, 2 * 1000); + String className = this.getClass().getName(); StringBuilder v = new StringBuilder(className); while (v.length() < 1000) { Index: src/contrib/hbase/src/test/org/apache/hadoop/hbase/StaticTestEnvironment.java =================================================================== --- src/contrib/hbase/src/test/org/apache/hadoop/hbase/StaticTestEnvironment.java (revision 581709) +++ src/contrib/hbase/src/test/org/apache/hadoop/hbase/StaticTestEnvironment.java (working copy) @@ -86,25 +86,25 @@ } else if(value.equalsIgnoreCase("WARN")) { logLevel = Level.WARN; } + } - ConsoleAppender consoleAppender = null; - for(Enumeration e = rootLogger.getAllAppenders(); - e.hasMoreElements();) { + ConsoleAppender consoleAppender = null; + for(Enumeration e = rootLogger.getAllAppenders(); + e.hasMoreElements();) { - Appender a = e.nextElement(); - if(a instanceof ConsoleAppender) { - consoleAppender = (ConsoleAppender)a; - break; - } + Appender a = e.nextElement(); + if(a instanceof ConsoleAppender) { + consoleAppender = (ConsoleAppender)a; + break; } - if(consoleAppender != null) { - Layout layout = consoleAppender.getLayout(); - if(layout instanceof PatternLayout) { - PatternLayout consoleLayout = (PatternLayout)layout; - consoleLayout.setConversionPattern("%d %-5p [%t] %l: %m%n"); - } + } + if(consoleAppender != null) { + Layout layout = consoleAppender.getLayout(); + if(layout instanceof PatternLayout) { + PatternLayout consoleLayout = (PatternLayout)layout; + consoleLayout.setConversionPattern("%d %-5p [%t] %l: %m%n"); } - } + } Logger.getLogger( HBaseTestCase.class.getPackage().getName()).setLevel(logLevel); } Index: src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/FSUtils.java =================================================================== --- src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/FSUtils.java (revision 581709) +++ src/contrib/hbase/src/java/org/apache/hadoop/hbase/util/FSUtils.java (working copy) @@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.util; import java.io.IOException; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -59,11 +58,11 @@ } catch (IOException e) { exception = e.getMessage(); } - LOG.info("Failed file system available test. Thread: " + - Thread.currentThread().getName() + ": " + exception); try { if (!available) { + LOG.info("Failed file system available test. Thread: " + + Thread.currentThread().getName() + ": " + exception); fs.close(); }