From 4b07abcfec6924cc690219fedea0b35166630f90 Mon Sep 17 00:00:00 2001 From: chenheng Date: Tue, 22 Dec 2015 18:29:31 +0800 Subject: [PATCH] HBASE-14684 Try to remove all MiniMapReduceCluster in unit tests --- .../org/apache/hadoop/hbase/HBaseCommonTestingUtility.java | 6 ++++++ .../hadoop/hbase/mapred/TestTableSnapshotInputFormat.java | 4 ++-- .../hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java | 7 +++---- .../hbase/mapreduce/TableSnapshotInputFormatTestBase.java | 8 ++++---- .../org/apache/hadoop/hbase/mapreduce/TestCellCounter.java | 3 +-- .../java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java | 3 +-- .../apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java | 11 ++++------- .../apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java | 11 +++-------- .../apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java | 3 +-- .../java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java | 3 +-- .../org/apache/hadoop/hbase/mapreduce/TestImportExport.java | 3 +-- .../hbase/mapreduce/TestImportTSVWithOperationAttributes.java | 4 +--- .../apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java | 3 +-- .../java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java | 3 +-- .../hadoop/hbase/mapreduce/TestMultiTableInputFormat.java | 7 +++---- .../hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java | 3 +-- .../org/apache/hadoop/hbase/mapreduce/TestRowCounter.java | 3 +-- .../java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java | 3 +-- .../apache/hadoop/hbase/mapreduce/TestTableInputFormat.java | 4 +--- .../apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java | 3 +-- .../hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java | 4 ++-- .../apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java | 3 +-- .../java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java | 3 +-- 23 files changed, 42 insertions(+), 63 deletions(-) diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseCommonTestingUtility.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseCommonTestingUtility.java index 18ca35c..b7361bf 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseCommonTestingUtility.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseCommonTestingUtility.java @@ -176,6 +176,12 @@ public class HBaseCommonTestingUtility { return new Path(PathName); } + public Path getRandomDir() { + String randomStr = UUID.randomUUID().toString(); + Path testPath = new Path(getBaseTestDir(), randomStr); + return testPath; + } + /** * @param dir Directory to delete * @return True if we deleted it. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java index 29c37c5..c3771c5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableSnapshotInputFormat.java @@ -103,7 +103,7 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa try { createTableAndSnapshot(UTIL, tableName, snapshotName, getStartRow(), getEndRow(), 1); JobConf job = new JobConf(UTIL.getConfiguration()); - Path tmpTableDir = UTIL.getDataTestDirOnTestFS(snapshotName); + Path tmpTableDir = UTIL.getRandomDir(); TableMapReduceUtil.initTableSnapshotMapJob(snapshotName, COLUMNS, TestTableSnapshotMapper.class, ImmutableBytesWritable.class, @@ -166,7 +166,7 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa util, tableName, snapshotName, getStartRow(), getEndRow(), numRegions); JobConf job = new JobConf(util.getConfiguration()); - Path tmpTableDir = util.getDataTestDirOnTestFS(snapshotName); + Path tmpTableDir = util.getRandomDir(); TableMapReduceUtil.initTableSnapshotMapJob(snapshotName, COLUMNS, TestTableSnapshotMapper.class, ImmutableBytesWritable.class, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java index eaedebf..174cc1f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java @@ -78,6 +78,7 @@ public abstract class MultiTableInputFormatTestBase { public static void setUpBeforeClass() throws Exception { // switch TIF to log at DEBUG level TEST_UTIL.enableDebug(MultiTableInputFormatBase.class); + TEST_UTIL.setJobWithoutMRCluster(); // start mini hbase cluster TEST_UTIL.startMiniCluster(3); // create and fill table @@ -92,13 +93,10 @@ public abstract class MultiTableInputFormatTestBase { } } } - // start MR cluster - TEST_UTIL.startMiniMapReduceCluster(); } @AfterClass public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniMapReduceCluster(); TEST_UTIL.shutdownMiniCluster(); } @@ -270,7 +268,8 @@ public abstract class MultiTableInputFormatTestBase { initJob(scans, job); job.setReducerClass(ScanReducer.class); job.setNumReduceTasks(1); // one to get final "first" and "last" key - FileOutputFormat.setOutputPath(job, new Path(job.getJobName())); + FileOutputFormat.setOutputPath(job, + new Path(TEST_UTIL.getDataTestDirOnTestFS(), job.getJobName())); LOG.info("Started " + job.getJobName()); job.waitForCompletion(true); assertTrue(job.isSuccessful()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java index 3808806..1e57b86 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.io.HFileLink; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.master.snapshot.SnapshotManager; +import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; import org.apache.hadoop.hbase.regionserver.StoreFileInfo; import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; import org.apache.hadoop.hbase.util.Bytes; @@ -60,6 +61,7 @@ public abstract class TableSnapshotInputFormatTestBase { public void setupCluster() throws Exception { setupConf(UTIL.getConfiguration()); + UTIL.setJobWithoutMRCluster(); UTIL.startMiniCluster(NUM_REGION_SERVERS, true); rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); fs = rootDir.getFileSystem(UTIL.getConfiguration()); @@ -121,7 +123,7 @@ public abstract class TableSnapshotInputFormatTestBase { try { createTableAndSnapshot(UTIL, tableName, snapshotName, getStartRow(), getEndRow(), 1); - Path tmpTableDir = UTIL.getDataTestDirOnTestFS(snapshotName); + Path tmpTableDir = UTIL.getRandomDir(); testRestoreSnapshotDoesNotCreateBackRefLinksInit(tableName, snapshotName,tmpTableDir); @@ -158,14 +160,12 @@ public abstract class TableSnapshotInputFormatTestBase { protected void testWithMapReduce(HBaseTestingUtility util, String snapshotName, int numRegions, int expectedNumSplits, boolean shutdownCluster) throws Exception { setupCluster(); - util.startMiniMapReduceCluster(); try { - Path tableDir = util.getDataTestDirOnTestFS(snapshotName); + Path tableDir = util.getRandomDir(); TableName tableName = TableName.valueOf("testWithMapReduce"); testWithMapReduceImpl(util, tableName, snapshotName, tableDir, numRegions, expectedNumSplits, shutdownCluster); } finally { - util.shutdownMiniMapReduceCluster(); tearDownCluster(); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java index e3d03b8..41b76ba 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java @@ -60,15 +60,14 @@ public class TestCellCounter { @BeforeClass public static void beforeClass() throws Exception { + UTIL.setJobWithoutMRCluster(); UTIL.startMiniCluster(); - UTIL.startMiniMapReduceCluster(); FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem()); FileUtil.fullyDelete(new File(OUTPUT_DIR)); } @AfterClass public static void afterClass() throws Exception { - UTIL.shutdownMiniMapReduceCluster(); UTIL.shutdownMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java index 5492938..b8ad5be 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java @@ -62,13 +62,12 @@ public class TestCopyTable { @BeforeClass public static void beforeClass() throws Exception { + TEST_UTIL.setJobWithoutMRCluster(); TEST_UTIL.startMiniCluster(3); - TEST_UTIL.startMiniMapReduceCluster(); } @AfterClass public static void afterClass() throws Exception { - TEST_UTIL.shutdownMiniMapReduceCluster(); TEST_UTIL.shutdownMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java index 2bdcbc1..b4bae99 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java @@ -390,6 +390,7 @@ public class TestHFileOutputFormat { byte[][] splitKeys = generateRandomSplitKeys(4); HBaseAdmin admin = null; try { + util.setJobWithoutMRCluster(); util.startMiniCluster(); Path testDir = util.getDataTestDirOnTestFS("testLocalMRIncrementalLoad"); admin = util.getHBaseAdmin(); @@ -403,7 +404,6 @@ public class TestHFileOutputFormat { assertEquals("Should make 5 regions", numRegions, 5); // Generate the bulk load files - util.startMiniMapReduceCluster(); runIncrementalPELoad(conf, table, testDir); // This doesn't write into the table, just makes files assertEquals("HFOF should not touch actual table", @@ -472,7 +472,6 @@ public class TestHFileOutputFormat { tableDigestBefore, util.checksumRows(table)); } finally { if (admin != null) admin.close(); - util.shutdownMiniMapReduceCluster(); util.shutdownMiniCluster(); } } @@ -919,6 +918,7 @@ public class TestHFileOutputFormat { generateRandomStartKeys(5); try { + util.setJobWithoutMRCluster(); util.startMiniCluster(); final FileSystem fs = util.getDFSCluster().getFileSystem(); HBaseAdmin admin = new HBaseAdmin(conf); @@ -935,7 +935,6 @@ public class TestHFileOutputFormat { // Generate two bulk load files conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude", true); - util.startMiniMapReduceCluster(); for (int i = 0; i < 2; i++) { Path testDir = util.getDataTestDirOnTestFS("testExcludeAllFromMinorCompaction_" + i); @@ -974,7 +973,6 @@ public class TestHFileOutputFormat { }, 5000); } finally { - util.shutdownMiniMapReduceCluster(); util.shutdownMiniCluster(); } } @@ -986,9 +984,10 @@ public class TestHFileOutputFormat { generateRandomStartKeys(5); try { + util.setJobWithoutMRCluster(); util.startMiniCluster(); Path testDir = util.getDataTestDirOnTestFS("testExcludeMinorCompaction"); - final FileSystem fs = util.getDFSCluster().getFileSystem(); + final FileSystem fs = util.getTestFileSystem(); HBaseAdmin admin = new HBaseAdmin(conf); HTable table = util.createTable(TABLE_NAME, FAMILIES); assertEquals("Should start with empty table", 0, util.countRows(table)); @@ -1015,7 +1014,6 @@ public class TestHFileOutputFormat { // Generate a bulk load file with more rows conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude", true); - util.startMiniMapReduceCluster(); runIncrementalPELoad(conf, table, testDir); // Perform the actual load @@ -1051,7 +1049,6 @@ public class TestHFileOutputFormat { }, 5000); } finally { - util.shutdownMiniMapReduceCluster(); util.shutdownMiniCluster(); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java index e55affc..bb1a073 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java @@ -421,6 +421,7 @@ public class TestHFileOutputFormat2 { for (int i = 0; i < hostCount; ++i) { hostnames[i] = "datanode_" + i; } + util.setJobWithoutMRCluster(); util.startMiniCluster(1, hostCount, hostnames); HTable table = util.createTable(TABLE_NAME, FAMILIES, splitKeys); @@ -431,7 +432,6 @@ public class TestHFileOutputFormat2 { assertEquals("Should make " + regionNum + " regions", numRegions, regionNum); // Generate the bulk load files - util.startMiniMapReduceCluster(); runIncrementalPELoad(conf, table.getTableDescriptor(), table.getRegionLocator(), testDir); // This doesn't write into the table, just makes files assertEquals("HFOF should not touch actual table", 0, util.countRows(table)); @@ -511,7 +511,6 @@ public class TestHFileOutputFormat2 { } finally { testDir.getFileSystem(conf).delete(testDir, true); util.deleteTable(TABLE_NAME); - util.shutdownMiniMapReduceCluster(); util.shutdownMiniCluster(); } } @@ -956,7 +955,7 @@ public class TestHFileOutputFormat2 { Configuration conf = util.getConfiguration(); conf.setInt("hbase.hstore.compaction.min", 2); generateRandomStartKeys(5); - + util.setJobWithoutMRCluster(); util.startMiniCluster(); try (Connection conn = ConnectionFactory.createConnection(); Admin admin = conn.getAdmin()) { @@ -974,7 +973,6 @@ public class TestHFileOutputFormat2 { // Generate two bulk load files conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude", true); - util.startMiniMapReduceCluster(); for (int i = 0; i < 2; i++) { Path testDir = util.getDataTestDirOnTestFS("testExcludeAllFromMinorCompaction_" + i); @@ -1016,7 +1014,6 @@ public class TestHFileOutputFormat2 { }, 5000); } finally { - util.shutdownMiniMapReduceCluster(); util.shutdownMiniCluster(); } } @@ -1026,7 +1023,7 @@ public class TestHFileOutputFormat2 { Configuration conf = util.getConfiguration(); conf.setInt("hbase.hstore.compaction.min", 2); generateRandomStartKeys(5); - + util.setJobWithoutMRCluster(); util.startMiniCluster(); try (Connection conn = ConnectionFactory.createConnection(conf); Admin admin = conn.getAdmin()){ @@ -1058,7 +1055,6 @@ public class TestHFileOutputFormat2 { // Generate a bulk load file with more rows conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude", true); - util.startMiniMapReduceCluster(); RegionLocator regionLocator = conn.getRegionLocator(TABLE_NAME); runIncrementalPELoad(conf, table.getTableDescriptor(), regionLocator, testDir); @@ -1098,7 +1094,6 @@ public class TestHFileOutputFormat2 { }, 5000); } finally { - util.shutdownMiniMapReduceCluster(); util.shutdownMiniCluster(); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java index 220bc02..4354da5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java @@ -34,13 +34,12 @@ public class TestHRegionPartitioner { @BeforeClass public static void beforeClass() throws Exception { + UTIL.setJobWithoutMRCluster(); UTIL.startMiniCluster(); - UTIL.startMiniMapReduceCluster(); } @AfterClass public static void afterClass() throws Exception { - UTIL.shutdownMiniMapReduceCluster(); UTIL.shutdownMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java index 762f530..eae33a6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java @@ -56,13 +56,12 @@ public class TestHashTable { @BeforeClass public static void beforeClass() throws Exception { + TEST_UTIL.setJobWithoutMRCluster(); TEST_UTIL.startMiniCluster(3); - TEST_UTIL.startMiniMapReduceCluster(); } @AfterClass public static void afterClass() throws Exception { - TEST_UTIL.shutdownMiniMapReduceCluster(); TEST_UTIL.shutdownMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java index 2e06fab..2154f99 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java @@ -107,15 +107,14 @@ public class TestImportExport { public static void beforeClass() throws Exception { // Up the handlers; this test needs more than usual. UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10); + UTIL.setJobWithoutMRCluster(); UTIL.startMiniCluster(); - UTIL.startMiniMapReduceCluster(); FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(FileSystem.get(UTIL.getConfiguration())).toString(); } @AfterClass public static void afterClass() throws Exception { - UTIL.shutdownMiniMapReduceCluster(); UTIL.shutdownMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java index eba843e..acb0e7c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java @@ -102,14 +102,12 @@ public class TestImportTSVWithOperationAttributes implements Configurable { conf = util.getConfiguration(); conf.set("hbase.coprocessor.master.classes", OperationAttributesTestController.class.getName()); conf.set("hbase.coprocessor.region.classes", OperationAttributesTestController.class.getName()); + util.setJobWithoutMRCluster(); util.startMiniCluster(); - Admin admin = new HBaseAdmin(util.getConfiguration()); - util.startMiniMapReduceCluster(); } @AfterClass public static void releaseCluster() throws Exception { - util.shutdownMiniMapReduceCluster(); util.shutdownMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java index 268bba2..5add5d4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java @@ -88,13 +88,12 @@ public class TestImportTSVWithTTLs implements Configurable { // need this where the default hfile version is not 3 (i.e. 0.98) conf.setInt("hfile.format.version", 3); conf.set("hbase.coprocessor.region.classes", TTLCheckingObserver.class.getName()); + util.setJobWithoutMRCluster(); util.startMiniCluster(); - util.startMiniMapReduceCluster(); } @AfterClass public static void releaseCluster() throws Exception { - util.shutdownMiniMapReduceCluster(); util.shutdownMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java index 5adc04a..40c4b4d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java @@ -103,13 +103,12 @@ public class TestImportTsv implements Configurable { @BeforeClass public static void provisionCluster() throws Exception { + util.setJobWithoutMRCluster(); util.startMiniCluster(); - util.startMiniMapReduceCluster(); } @AfterClass public static void releaseCluster() throws Exception { - util.shutdownMiniMapReduceCluster(); util.shutdownMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableInputFormat.java index a54df08..8a4a244 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableInputFormat.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableInputFormat.java @@ -71,6 +71,7 @@ public class TestMultiTableInputFormat { // switch TIF to log at DEBUG level TEST_UTIL.enableDebug(MultiTableInputFormat.class); TEST_UTIL.enableDebug(MultiTableInputFormatBase.class); + TEST_UTIL.setJobWithoutMRCluster(); // start mini hbase cluster TEST_UTIL.startMiniCluster(3); // create and fill table @@ -81,13 +82,10 @@ public class TestMultiTableInputFormat { TEST_UTIL.loadTable(table, INPUT_FAMILY, false); } } - // start MR cluster - TEST_UTIL.startMiniMapReduceCluster(); } @AfterClass public static void tearDownAfterClass() throws Exception { - TEST_UTIL.shutdownMiniMapReduceCluster(); TEST_UTIL.shutdownMiniCluster(); } @@ -242,7 +240,8 @@ public class TestMultiTableInputFormat { ImmutableBytesWritable.class, ImmutableBytesWritable.class, job); job.setReducerClass(ScanReducer.class); job.setNumReduceTasks(1); // one to get final "first" and "last" key - FileOutputFormat.setOutputPath(job, new Path(job.getJobName())); + FileOutputFormat.setOutputPath(job, + new Path(TEST_UTIL.getDataTestDirOnTestFS(), job.getJobName())); LOG.info("Started " + job.getJobName()); job.waitForCompletion(true); assertTrue(job.isSuccessful()); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java index 6b2ee75..9a81990 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java @@ -72,18 +72,17 @@ public class TestMultithreadedTableMapper { public static void beforeClass() throws Exception { // Up the handlers; this test needs more than usual. UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10); + UTIL.setJobWithoutMRCluster(); UTIL.startMiniCluster(); HTable table = UTIL.createMultiRegionTable(MULTI_REGION_TABLE_NAME, new byte[][] { INPUT_FAMILY, OUTPUT_FAMILY }); UTIL.loadTable(table, INPUT_FAMILY, false); - UTIL.startMiniMapReduceCluster(); UTIL.waitUntilAllRegionsAssigned(MULTI_REGION_TABLE_NAME); } @AfterClass public static void afterClass() throws Exception { - UTIL.shutdownMiniMapReduceCluster(); UTIL.shutdownMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java index 817e7a9..0e04c67 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java @@ -80,8 +80,8 @@ public class TestRowCounter { @BeforeClass public static void setUpBeforeClass() throws Exception { + TEST_UTIL.setJobWithoutMRCluster(); TEST_UTIL.startMiniCluster(); - TEST_UTIL.startMiniMapReduceCluster(); Table table = TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME), Bytes.toBytes(COL_FAM)); writeRows(table); table.close(); @@ -94,7 +94,6 @@ public class TestRowCounter { public static void tearDownAfterClass() throws Exception { TEST_UTIL.shutdownMiniCluster(); - TEST_UTIL.shutdownMiniMapReduceCluster(); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java index 2f1bee3..1b356e6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java @@ -64,13 +64,12 @@ public class TestSyncTable { @BeforeClass public static void beforeClass() throws Exception { + TEST_UTIL.setJobWithoutMRCluster(); TEST_UTIL.startMiniCluster(3); - TEST_UTIL.startMiniMapReduceCluster(); } @AfterClass public static void afterClass() throws Exception { - TEST_UTIL.shutdownMiniMapReduceCluster(); TEST_UTIL.shutdownMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java index c53510f..d404833 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java @@ -77,20 +77,18 @@ public class TestTableInputFormat { private static final Log LOG = LogFactory.getLog(TestTableInputFormat.class); private final static HBaseTestingUtility UTIL = new HBaseTestingUtility(); - private static MiniMRCluster mrCluster; static final byte[] FAMILY = Bytes.toBytes("family"); private static final byte[][] columns = new byte[][] { FAMILY }; @BeforeClass public static void beforeClass() throws Exception { + UTIL.setJobWithoutMRCluster(); UTIL.startMiniCluster(); - mrCluster = UTIL.startMiniMapReduceCluster(); } @AfterClass public static void afterClass() throws Exception { - UTIL.shutdownMiniMapReduceCluster(); UTIL.shutdownMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java index 8aed588..022d4c9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java @@ -79,17 +79,16 @@ public abstract class TestTableMapReduceBase { @BeforeClass public static void beforeClass() throws Exception { + UTIL.setJobWithoutMRCluster(); UTIL.startMiniCluster(); HTable table = UTIL.createMultiRegionTable(MULTI_REGION_TABLE_NAME, new byte[][] { INPUT_FAMILY, OUTPUT_FAMILY }); UTIL.loadTable(table, INPUT_FAMILY, false); - UTIL.startMiniMapReduceCluster(); } @AfterClass public static void afterClass() throws Exception { - UTIL.shutdownMiniMapReduceCluster(); UTIL.shutdownMiniCluster(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java index c3dcd8f..1b80590 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableSnapshotInputFormat.java @@ -155,7 +155,7 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa try { createTableAndSnapshot(UTIL, tableName, snapshotName, getStartRow(), getEndRow(), 1); Job job = new Job(UTIL.getConfiguration()); - Path tmpTableDir = UTIL.getDataTestDirOnTestFS(snapshotName); + Path tmpTableDir = UTIL.getRandomDir(); TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName, new Scan(), TestTableSnapshotMapper.class, ImmutableBytesWritable.class, @@ -196,7 +196,7 @@ public class TestTableSnapshotInputFormat extends TableSnapshotInputFormatTestBa util, tableName, snapshotName, getStartRow(), getEndRow(), numRegions); Job job = new Job(util.getConfiguration()); - Path tmpTableDir = util.getDataTestDirOnTestFS(snapshotName); + Path tmpTableDir = util.getRandomDir(); Scan scan = new Scan(getStartRow(), getEndRow()); // limit the scan TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java index cd85756..207f256 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java @@ -83,6 +83,7 @@ public class TestTimeRangeMapRed { @BeforeClass public static void beforeClass() throws Exception { + UTIL.setJobWithoutMRCluster(); UTIL.startMiniCluster(); } @@ -167,7 +168,6 @@ public class TestTimeRangeMapRed { private void runTestOnTable() throws IOException, InterruptedException, ClassNotFoundException { - UTIL.startMiniMapReduceCluster(); Job job = null; try { job = new Job(UTIL.getConfiguration(), "test123"); @@ -184,7 +184,6 @@ public class TestTimeRangeMapRed { // TODO Auto-generated catch block e.printStackTrace(); } finally { - UTIL.shutdownMiniMapReduceCluster(); if (job != null) { FileUtil.fullyDelete( new File(job.getConfiguration().get("hadoop.tmp.dir"))); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java index d6929a2..343fc64 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java @@ -70,13 +70,12 @@ public class TestWALPlayer { @BeforeClass public static void beforeClass() throws Exception { + TEST_UTIL.setJobWithoutMRCluster(); cluster = TEST_UTIL.startMiniCluster(); - TEST_UTIL.startMiniMapReduceCluster(); } @AfterClass public static void afterClass() throws Exception { - TEST_UTIL.shutdownMiniMapReduceCluster(); TEST_UTIL.shutdownMiniCluster(); } -- 1.9.3 (Apple Git-50)