diff --git dev-support/hbasetests.sh dev-support/hbasetests.sh index de12c1b..e129bd4 100755 --- dev-support/hbasetests.sh +++ dev-support/hbasetests.sh @@ -307,11 +307,11 @@ echo "Small tests executed after $exeTime minutes" if (test $parallelMaven -gt 0) then echo "Running tests with two maven instances in parallel" - $mvnCommand -P nonParallelTests test -Dtest=$runList1 $args & + $mvnCommand -P localTests test -Dtest=$runList1 $args & #give some time to the fist process if there is anything to compile sleep 30 - $mvnCommand -P nonParallelTests test -Dtest=$runList2 $args + $mvnCommand -P localTests test -Dtest=$runList2 $args #wait for forked process to finish wait @@ -326,14 +326,14 @@ then if (test $runAllTests -eq 1 && test ${#flakyTests} -gt 5) then echo "Running flaky tests" - $mvnCommand -P nonParallelTests test -Dtest=$flakyTests $args + $mvnCommand -P localTests test -Dtest=$flakyTests $args cleanProcess exeTime=$(((`date +%s` - $startTime)/60)) echo "Flaky tests executed after $exeTime minutes" fi else echo "Running tests with a single maven instance, no parallelization" - $mvnCommand -P nonParallelTests test -Dtest=$runList1,$runList2,$flakyTests $args + $mvnCommand -P localTests test -Dtest=$runList1,$runList2,$flakyTests $args cleanProcess exeTime=$(((`date +%s` - $startTime)/60)) echo "Single maven instance tests executed after $exeTime minutes" @@ -420,7 +420,7 @@ then if (test $replayFailed -gt 0) then echo "Replaying all tests that failed" - $mvnCommand -P nonParallelTests test -Dtest=$replayList $args + $mvnCommand -P localTests test -Dtest=$replayList $args echo "Replaying done" fi fi diff --git pom.xml pom.xml index 10dc81d..a828f2c 100644 --- pom.xml +++ pom.xml @@ -329,9 +329,7 @@ --> org.apache.maven.surefire - surefire-junit47 - + ${surefire.provider} ${surefire.version} @@ -339,7 +337,7 @@ 900 -enableassertions -Xmx1900m -Djava.security.egd=file:/dev/./urandom - true + false @@ -351,7 +349,23 @@ org.apache.maven.plugins maven-failsafe-plugin ${surefire.version} + + + org.apache.maven.surefire + ${surefire.provider} + ${surefire.version} + + + + ${integrationtest.include} + + + ${unittest.include} + **/*$* + ${test.exclude.pattern} + + true ${env.LD_LIBRARY_PATH}:${project.build.directory}/nativelib ${env.DYLD_LIBRARY_PATH}:${project.build.directory}/nativelib @@ -605,13 +619,13 @@ org.apache.maven.plugins maven-surefire-plugin - ${surefire.skipfirstPartTests} + ${surefire.skipFirstPart} ${surefire.firstPartForkMode} ${surefire.firstPartParallel} false ${surefire.firstPartThreadCount} ${surefire.firstPartGroups} - ${surefire.hasSecondPart} + false @@ -634,8 +648,7 @@ maven-failsafe-plugin false - always - org.apache.hadoop.hbase.LargeTests + always @@ -872,22 +885,24 @@ 0.91.0 ${project.artifactId}-${project.version} + + + **/Test*.java + **/IntegrationTest*.java 2.11-TRUNK-HBASE-2 - - always - none - 1 + surefire-junit47 - + false - true - false + false - - + once + none + 1 + + org.apache.hadoop.hbase.SmallTests + org.apache.hadoop.hbase.MediumTests @@ -901,6 +916,7 @@ * javax.xml.stream:stax-api in favour of stax:stax-api --> + com.google.guava @@ -1886,6 +1902,10 @@ once none 1 + + false + true + @@ -1912,6 +1932,7 @@ false + always false true org.apache.hadoop.hbase.MediumTests @@ -1925,6 +1946,7 @@ false + always false true org.apache.hadoop.hbase.LargeTests @@ -1944,11 +1966,10 @@ false false - true org.apache.hadoop.hbase.SmallTests org.apache.hadoop.hbase.MediumTests - + runAllTests @@ -1962,11 +1983,37 @@ false false - true org.apache.hadoop.hbase.SmallTests org.apache.hadoop.hbase.MediumTests,org.apache.hadoop.hbase.LargeTests - + + + + skipSurefireTests + + false + + + true + true + + + + + localTests + + false + + + surefire-junit4 + 2.10 + + always + false + true + + + diff --git src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java index 9d26994..e5d19a5 100644 --- src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java +++ src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java @@ -43,7 +43,7 @@ import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.InternalScanner; -import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.*; import org.apache.hadoop.hdfs.MiniDFSCluster; /** @@ -118,6 +118,10 @@ public abstract class HBaseTestCase extends TestCase { */ @Override protected void setUp() throws Exception { + LOG.info("The current edge manager is: "+ + EnvironmentEdgeManager.getDelegate().getClass()); + + super.setUp(); localfs = (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0); diff --git src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java index 56a4e89..3141f9b 100644 --- src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java +++ src/test/java/org/apache/hadoop/hbase/ipc/TestDelayedRpc.java @@ -30,9 +30,9 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; +import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.SmallTests; +import org.apache.hadoop.hbase.*; import org.apache.hadoop.hbase.ipc.VersionedProtocol; import org.apache.log4j.AppenderSkeleton; import org.apache.log4j.Logger; @@ -45,7 +45,7 @@ import org.junit.experimental.categories.Category; * be delayed. Check that the last two, which are undelayed, return before the * first one. */ -@Category(SmallTests.class) +@Category(MediumTests.class) // Fails sometimes with small tests public class TestDelayedRpc { public static RpcServer rpcServer; @@ -233,11 +233,15 @@ public class TestDelayedRpc { @Override public void run() { - Integer result = new Integer(server.test(delay)); - if (results != null) { - synchronized (results) { - results.add(result); + try { + Integer result = new Integer(server.test(delay)); + if (results != null) { + synchronized (results) { + results.add(result); + } } + } catch (Exception e) { + fail("Unexpected exception: "+e.getMessage()); } } } diff --git src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java index 5113c1f..92ab06c 100644 --- src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java @@ -388,6 +388,12 @@ public class TestStore extends TestCase { assertEquals(oldValue, Bytes.toLong(results.get(1).getValue())); } + @Override + protected void tearDown() throws Exception { + super.tearDown(); + EnvironmentEdgeManagerTestHelper.reset(); + } + public void testICV_negMemstoreSize() throws IOException { init(this.getName()); diff --git src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java index d0cda62..07163f2 100644 --- src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java @@ -41,7 +41,8 @@ import java.util.NavigableSet; import java.util.TreeSet; import static org.apache.hadoop.hbase.regionserver.KeyValueScanFixture.scanFixture; -@Category(SmallTests.class) +// Can't be small as it plays with EnvironmentEdgeManager +@Category(MediumTests.class) public class TestStoreScanner extends TestCase { private static final String CF_STR = "cf"; final byte [] CF = Bytes.toBytes(CF_STR); @@ -502,69 +503,73 @@ public class TestStoreScanner extends TestCase { } public void testDeleteMarkerLongevity() throws Exception { - final long now = System.currentTimeMillis(); - EnvironmentEdgeManagerTestHelper.injectEdge(new EnvironmentEdge() { - public long currentTimeMillis() { - return now; - } - }); - KeyValue [] kvs = new KeyValue[] { + try { + final long now = System.currentTimeMillis(); + EnvironmentEdgeManagerTestHelper.injectEdge(new EnvironmentEdge() { + public long currentTimeMillis() { + return now; + } + }); + KeyValue[] kvs = new KeyValue[]{ /*0*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, - now-100, KeyValue.Type.DeleteFamily), // live + now - 100, KeyValue.Type.DeleteFamily), // live /*1*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, - now-1000, KeyValue.Type.DeleteFamily), // expired - /*2*/ KeyValueTestUtil.create("R1", "cf", "a", now-50, - KeyValue.Type.Put, "v3"), // live - /*3*/ KeyValueTestUtil.create("R1", "cf", "a", now-55, - KeyValue.Type.Delete, "dontcare"), // live - /*4*/ KeyValueTestUtil.create("R1", "cf", "a", now-55, - KeyValue.Type.Put, "deleted-version v2"), // deleted - /*5*/ KeyValueTestUtil.create("R1", "cf", "a", now-60, - KeyValue.Type.Put, "v1"), // live - /*6*/ KeyValueTestUtil.create("R1", "cf", "a", now-65, - KeyValue.Type.Put, "v0"), // max-version reached + now - 1000, KeyValue.Type.DeleteFamily), // expired + /*2*/ KeyValueTestUtil.create("R1", "cf", "a", now - 50, + KeyValue.Type.Put, "v3"), // live + /*3*/ KeyValueTestUtil.create("R1", "cf", "a", now - 55, + KeyValue.Type.Delete, "dontcare"), // live + /*4*/ KeyValueTestUtil.create("R1", "cf", "a", now - 55, + KeyValue.Type.Put, "deleted-version v2"), // deleted + /*5*/ KeyValueTestUtil.create("R1", "cf", "a", now - 60, + KeyValue.Type.Put, "v1"), // live + /*6*/ KeyValueTestUtil.create("R1", "cf", "a", now - 65, + KeyValue.Type.Put, "v0"), // max-version reached /*7*/ KeyValueTestUtil.create("R1", "cf", "a", - now-100, KeyValue.Type.DeleteColumn, "dont-care"), // max-version - /*8*/ KeyValueTestUtil.create("R1", "cf", "b", now-600, - KeyValue.Type.DeleteColumn, "dont-care"), //expired - /*9*/ KeyValueTestUtil.create("R1", "cf", "b", now-70, - KeyValue.Type.Put, "v2"), //live - /*10*/ KeyValueTestUtil.create("R1", "cf", "b", now-750, - KeyValue.Type.Put, "v1"), //expired - /*11*/ KeyValueTestUtil.create("R1", "cf", "c", now-500, - KeyValue.Type.Delete, "dontcare"), //expired - /*12*/ KeyValueTestUtil.create("R1", "cf", "c", now-600, - KeyValue.Type.Put, "v1"), //expired - /*13*/ KeyValueTestUtil.create("R1", "cf", "c", now-1000, - KeyValue.Type.Delete, "dontcare"), //expired - /*14*/ KeyValueTestUtil.create("R1", "cf", "d", now-60, - KeyValue.Type.Put, "expired put"), //live - /*15*/ KeyValueTestUtil.create("R1", "cf", "d", now-100, - KeyValue.Type.Delete, "not-expired delete"), //live - }; - List scanners = scanFixture(kvs); - Scan scan = new Scan(); - scan.setMaxVersions(2); - Store.ScanInfo scanInfo = new Store.ScanInfo(Bytes.toBytes("cf"), + now - 100, KeyValue.Type.DeleteColumn, "dont-care"), // max-version + /*8*/ KeyValueTestUtil.create("R1", "cf", "b", now - 600, + KeyValue.Type.DeleteColumn, "dont-care"), //expired + /*9*/ KeyValueTestUtil.create("R1", "cf", "b", now - 70, + KeyValue.Type.Put, "v2"), //live + /*10*/ KeyValueTestUtil.create("R1", "cf", "b", now - 750, + KeyValue.Type.Put, "v1"), //expired + /*11*/ KeyValueTestUtil.create("R1", "cf", "c", now - 500, + KeyValue.Type.Delete, "dontcare"), //expired + /*12*/ KeyValueTestUtil.create("R1", "cf", "c", now - 600, + KeyValue.Type.Put, "v1"), //expired + /*13*/ KeyValueTestUtil.create("R1", "cf", "c", now - 1000, + KeyValue.Type.Delete, "dontcare"), //expired + /*14*/ KeyValueTestUtil.create("R1", "cf", "d", now - 60, + KeyValue.Type.Put, "expired put"), //live + /*15*/ KeyValueTestUtil.create("R1", "cf", "d", now - 100, + KeyValue.Type.Delete, "not-expired delete"), //live + }; + List scanners = scanFixture(kvs); + Scan scan = new Scan(); + scan.setMaxVersions(2); + Store.ScanInfo scanInfo = new Store.ScanInfo(Bytes.toBytes("cf"), 0 /* minVersions */, 2 /* maxVersions */, 500 /* ttl */, false /* keepDeletedCells */, 200, /* timeToPurgeDeletes */ KeyValue.COMPARATOR); - StoreScanner scanner = + StoreScanner scanner = new StoreScanner(scan, scanInfo, - StoreScanner.ScanType.MAJOR_COMPACT, null, scanners, - HConstants.OLDEST_TIMESTAMP); - List results = new ArrayList(); - results = new ArrayList(); - assertEquals(true, scanner.next(results)); - assertEquals(kvs[0], results.get(0)); - assertEquals(kvs[2], results.get(1)); - assertEquals(kvs[3], results.get(2)); - assertEquals(kvs[5], results.get(3)); - assertEquals(kvs[9], results.get(4)); - assertEquals(kvs[14], results.get(5)); - assertEquals(kvs[15], results.get(6)); - assertEquals(7, results.size()); - } + StoreScanner.ScanType.MAJOR_COMPACT, null, scanners, + HConstants.OLDEST_TIMESTAMP); + List results = new ArrayList(); + results = new ArrayList(); + assertEquals(true, scanner.next(results)); + assertEquals(kvs[0], results.get(0)); + assertEquals(kvs[2], results.get(1)); + assertEquals(kvs[3], results.get(2)); + assertEquals(kvs[5], results.get(3)); + assertEquals(kvs[9], results.get(4)); + assertEquals(kvs[14], results.get(5)); + assertEquals(kvs[15], results.get(6)); + assertEquals(7, results.size()); + }finally{ + EnvironmentEdgeManagerTestHelper.reset(); + } + } } diff --git src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java index caf3d19..ccf6fa4 100644 --- src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java +++ src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java @@ -36,7 +36,8 @@ import org.junit.experimental.categories.Category; /** * Tests for {@link FSTableDescriptors}. */ -@Category(SmallTests.class) +// Do not support to be executed in he same JVM as other tests +@Category(MediumTests.class) public class TestFSTableDescriptors { private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); private static final Log LOG = LogFactory.getLog(TestFSTableDescriptors.class); diff --git src/test/java/org/apache/hadoop/hbase/util/TestIncrementingEnvironmentEdge.java src/test/java/org/apache/hadoop/hbase/util/TestIncrementingEnvironmentEdge.java index 56453c9..52fbfaa 100644 --- src/test/java/org/apache/hadoop/hbase/util/TestIncrementingEnvironmentEdge.java +++ src/test/java/org/apache/hadoop/hbase/util/TestIncrementingEnvironmentEdge.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.util; import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.SmallTests; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -29,7 +30,7 @@ import static junit.framework.Assert.assertEquals; * Tests that the incrementing environment edge increments time instead of using * the default. */ -@Category(MediumTests.class) +@Category(SmallTests.class) public class TestIncrementingEnvironmentEdge { @Test