diff --git a/pom.xml b/pom.xml index 53bbd64..794ae05 100644 --- a/pom.xml +++ b/pom.xml @@ -227,6 +227,15 @@ maven-clean-plugin 2.4 + + + + + build + + + org.apache.rat diff --git a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 3feaedd..9afed23 100644 --- a/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -48,20 +48,21 @@ import org.apache.hadoop.hbase.util.Writables; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.mapred.MiniMRCluster; +import org.apache.hadoop.security.UnixUserGroupInformation; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.zookeeper.ZooKeeper; /** * Facility for testing HBase. Added as tool to abet junit4 testing. Replaces * old HBaseTestCase and HBaseCluserTestCase functionality. * Create an instance and keep it around doing HBase testing. This class is - * meant to be your one-stop shop for anything you mind need testing. Manages - * one cluster at a time only. Depends on log4j on classpath and hbase-site.xml - * for logging and test-run configuration. It does not set logging levels nor - * make changes to configuration parameters. + * meant to be your one-stop shop for anything you might need testing. Manages + * one cluster at a time only. Depends on log4j being on classpath and + * hbase-site.xml for logging and test-run configuration. It does not set + * logging levels nor make changes to configuration parameters. */ public class HBaseTestingUtility { private final Log LOG = LogFactory.getLog(getClass()); - private final Configuration conf; private MiniZooKeeperCluster zkCluster = null; private MiniDFSCluster dfsCluster = null; @@ -71,6 +72,23 @@ public class HBaseTestingUtility { private File clusterTestBuildDir = null; private HBaseAdmin hbaseAdmin = null; + // Cache this. For some reason only works first time I get it. TODO: Figure + // out why. + private final static UserGroupInformation UGI; + static { + UGI = UserGroupInformation.getCurrentUGI(); + } + + /** + * System property key to get test directory value. + */ + public static final String TEST_DIRECTORY_KEY = "test.build.data"; + + /** + * Default parent directory for test output. + */ + public static final String DEFAULT_TEST_DIRECTORY = "target/build/data"; + public HBaseTestingUtility() { this(HBaseConfiguration.create()); } @@ -79,10 +97,6 @@ public class HBaseTestingUtility { this.conf = conf; } - /** System property key to get test directory value. - */ - public static final String TEST_DIRECTORY_KEY = "test.build.data"; - /** * @return Instance of Configuration. */ @@ -91,27 +105,38 @@ public class HBaseTestingUtility { } /** - * @return Where to write test data on local filesystem; usually build/test/data + * @return Where to write test data on local filesystem; usually + * {@link #DEFAULT_TEST_DIRECTORY} + * @see #setupClusterTestBuildDir() */ public static Path getTestDir() { - return new Path(System.getProperty(TEST_DIRECTORY_KEY, "target/test/data")); + return new Path(System.getProperty(TEST_DIRECTORY_KEY, + DEFAULT_TEST_DIRECTORY)); } /** - * Home our cluster in a dir under build/test. Give it a random name + * @param subdirName + * @return Path to a subdirectory named subdirName under + * {@link #getTestDir()}. + * @see #setupClusterTestBuildDir() + */ + public static Path getTestDir(final String subdirName) { + return new Path(getTestDir(), subdirName); + } + + /** + * Home our cluster in a dir under target/test. Give it a random name * so can have many concurrent clusters running if we need to. Need to * amend the test.build.data System property. Its what minidfscluster bases * it data dir on. Moding a System property is not the way to do concurrent * instances -- another instance could grab the temporary - * value unintentionally -- but not anything can do about it at moment; its - * how the minidfscluster works. + * value unintentionally -- but not anything can do about it at moment; + * single instance only is how the minidfscluster works. * @return The calculated cluster test build directory. */ File setupClusterTestBuildDir() { - String oldTestBuildDir = - System.getProperty(TEST_DIRECTORY_KEY, "build/test/data"); String randomStr = UUID.randomUUID().toString(); - String dirStr = oldTestBuildDir + "." + randomStr; + String dirStr = getTestDir(randomStr).toString(); File dir = new File(dirStr).getAbsoluteFile(); // Have it cleaned up on exit dir.deleteOnExit(); @@ -119,7 +144,7 @@ public class HBaseTestingUtility { } /** - * @throws IOException If cluster already running. + * @throws IOException If a cluster -- zk, dfs, or hbase -- already running. */ void isRunningCluster() throws IOException { if (this.clusterTestBuildDir == null) return; @@ -128,20 +153,50 @@ public class HBaseTestingUtility { } /** - * @param subdirName - * @return Path to a subdirectory named subdirName under - * {@link #getTestDir()}. + * Start a minidfscluster. + * @param servers How many DNs to start. + * @throws Exception + * @see {@link #shutdownMiniDFSCluster()} + * @return The mini dfs cluster created. */ - public Path getTestDir(final String subdirName) { - return new Path(getTestDir(), subdirName); + public MiniDFSCluster startMiniDFSCluster(int servers) throws Exception { + return startMiniDFSCluster(servers, null); } /** - * Start up a minicluster of hbase, dfs, and zookeeper. + * Start a minidfscluster. + * Can only create one. + * @param dir Where to home your dfs cluster. + * @param servers How many DNs to start. * @throws Exception + * @see {@link #shutdownMiniDFSCluster()} + * @return The mini dfs cluster created. */ - public void startMiniCluster() throws Exception { - startMiniCluster(1); + public MiniDFSCluster startMiniDFSCluster(int servers, final File dir) + throws Exception { + // This does the following to home the minidfscluster + // base_dir = new File(System.getProperty("test.build.data", "build/test/data"), "dfs/"); + // Some tests also do this: + // System.getProperty("test.cache.data", "build/test/cache"); + if (dir == null) this.clusterTestBuildDir = setupClusterTestBuildDir(); + else this.clusterTestBuildDir = dir; + System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestBuildDir.toString()); + System.setProperty("test.cache.data", this.clusterTestBuildDir.toString()); + this.dfsCluster = new MiniDFSCluster(12345, this.conf, servers, true, + true, true, null, null, null, null); + return this.dfsCluster; + } + + /** + * Shuts down instance created by call to {@link #startMiniDFSCluster(int, File)} + * or does nothing. + * @throws Exception + */ + public void shutdownMiniDFSCluster() throws Exception { + if (this.dfsCluster != null) { + // The below throws an exception per dn, AsynchronousCloseException. + this.dfsCluster.shutdown(); + } } /** @@ -149,13 +204,15 @@ public class HBaseTestingUtility { * @see #startMiniZKCluster() if you want zk + dfs + hbase mini cluster. * @throws Exception * @see #shutdownMiniZKCluster() + * @return zk cluster started. */ - public void startMiniZKCluster() throws Exception { - startMiniZKCluster(setupClusterTestBuildDir()); + public MiniZooKeeperCluster startMiniZKCluster() throws Exception { + return startMiniZKCluster(setupClusterTestBuildDir()); } - private void startMiniZKCluster(final File dir) throws Exception { + private MiniZooKeeperCluster startMiniZKCluster(final File dir) + throws Exception { if (this.zkCluster != null) { throw new IOException("Cluster already running at " + dir); } @@ -163,9 +220,12 @@ public class HBaseTestingUtility { int clientPort = this.zkCluster.startup(dir); this.conf.set("hbase.zookeeper.property.clientPort", Integer.toString(clientPort)); + return this.zkCluster; } /** + * Shuts down zk cluster created by call to {@link #startMiniZKCluster(File)} + * or does nothing. * @throws IOException * @see #startMiniZKCluster() */ @@ -174,44 +234,50 @@ public class HBaseTestingUtility { } /** - * Start up a minicluster of hbase, optinally dfs, and zookeeper. + * Start up a minicluster of hbase, dfs, and zookeeper. + * @throws Exception + * @return Mini hbase cluster instance created. + * @see {@link #shutdownMiniDFSCluster()} + */ + public MiniHBaseCluster startMiniCluster() throws Exception { + return startMiniCluster(1); + } + + /** + * Start up a minicluster of hbase, optionally dfs, and zookeeper. * Modifies Configuration. Homes the cluster data directory under a random * subdirectory in a directory under System property test.build.data. + * Directory is cleaned up on exit. * @param servers Number of servers to start up. We'll start this many * datanodes and regionservers. If servers is > 1, then make sure * hbase.regionserver.info.port is -1 (i.e. no ui per regionserver) otherwise * bind errors. * @throws Exception * @see {@link #shutdownMiniCluster()} + * @return Mini hbase cluster instance created. */ - public void startMiniCluster(final int servers) + public MiniHBaseCluster startMiniCluster(final int servers) throws Exception { LOG.info("Starting up minicluster"); // If we already put up a cluster, fail. isRunningCluster(); - String oldBuildTestDir = - System.getProperty(TEST_DIRECTORY_KEY, "build/test/data"); + // Make a new random dir to home everything in. Set it as system property. + // minidfs reads home from system property. this.clusterTestBuildDir = setupClusterTestBuildDir(); - - // Set our random dir while minidfscluster is being constructed. System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestBuildDir.getPath()); // Bring up mini dfs cluster. This spews a bunch of warnings about missing - // scheme. TODO: fix. - // Complaints are 'Scheme is undefined for build/test/data/dfs/name1'. - this.dfsCluster = new MiniDFSCluster(0, this.conf, servers, true, - true, true, null, null, null, null); - // Restore System property. minidfscluster accesses content of - // the TEST_DIRECTORY_KEY to make bad blocks, a feature we are not using, - // but otherwise, just in constructor. - System.setProperty(TEST_DIRECTORY_KEY, oldBuildTestDir); + // scheme. Complaints are 'Scheme is undefined for build/test/data/dfs/name1'. + startMiniDFSCluster(servers, this.clusterTestBuildDir); // Mangle conf so fs parameter points to minidfs we just started up FileSystem fs = this.dfsCluster.getFileSystem(); this.conf.set("fs.defaultFS", fs.getUri().toString()); + // Do old style too just to be safe. + this.conf.set("fs.default.name", fs.getUri().toString()); this.dfsCluster.waitClusterUp(); - - // It could be created before the cluster - if(this.zkCluster == null) { + + // Start up a zk cluster. + if (this.zkCluster == null) { startMiniZKCluster(this.clusterTestBuildDir); } @@ -226,6 +292,7 @@ public class HBaseTestingUtility { ResultScanner s = t.getScanner(new Scan()); while (s.next() != null) continue; LOG.info("Minicluster is up"); + return this.hbaseCluster; } /** @@ -503,25 +570,6 @@ public class HBaseTestingUtility { } /** - * Removes all rows from the .META. in preparation to add custom ones. - * - * @throws IOException When removing the rows fails. - */ - private void emptyMetaTable() throws IOException { - HTable t = new HTable(this.conf, HConstants.META_TABLE_NAME); - ArrayList deletes = new ArrayList(); - ResultScanner s = t.getScanner(new Scan()); - for (Result result : s) { - LOG.info("emptyMetaTable: remove row -> " + - Bytes.toStringBinary(result.getRow())); - Delete del = new Delete(result.getRow()); - deletes.add(del); - } - s.close(); - t.delete(deletes); - } - - /** * Starts a MiniMRCluster with a default number of * TaskTracker's. * @@ -688,4 +736,23 @@ public class HBaseTestingUtility { public MiniDFSCluster getDFSCluster() { return dfsCluster; } -} + + /** + * Create a copy of the passed configuration laden with a new user. Use it + * to do things like get a new FileSystem instance. + * @param c + * @param index Some unique number used to make a unique user. + * @return Copy of c with new user stuff loaded into it. + * @throws IOException + */ + public static Configuration setDifferentUser(final Configuration c, + final int index) + throws IOException { + Configuration c2 = new Configuration(c); + String username = UGI.getUserName() + ".hrs." + index; + UnixUserGroupInformation.saveToConf(c2, + UnixUserGroupInformation.UGI_PROPERTY_NAME, + new UnixUserGroupInformation(username, new String[]{"supergroup"})); + return c2; + } +} \ No newline at end of file diff --git a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java index 92f6577..a26dd50 100644 --- a/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java +++ b/src/test/java/org/apache/hadoop/hbase/MiniHBaseCluster.java @@ -20,7 +20,6 @@ package org.apache.hadoop.hbase; import java.io.IOException; -import java.net.BindException; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -36,10 +35,7 @@ import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.Threads; -import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.io.MapWritable; -import org.apache.hadoop.security.UnixUserGroupInformation; -import org.apache.hadoop.security.UserGroupInformation; /** * This class creates a single process HBase cluster. @@ -49,13 +45,6 @@ import org.apache.hadoop.security.UserGroupInformation; */ public class MiniHBaseCluster implements HConstants { static final Log LOG = LogFactory.getLog(MiniHBaseCluster.class.getName()); - // Cache this. For some reason only works first time I get it. TODO: Figure - // out why. - private final static UserGroupInformation UGI; - static { - UGI = UserGroupInformation.getCurrentUGI(); - } - private Configuration conf; public LocalHBaseCluster hbaseCluster; @@ -133,28 +122,7 @@ public class MiniHBaseCluster implements HConstants { public MiniHBaseClusterRegionServer(Configuration conf) throws IOException { - super(setDifferentUser(conf)); - } - - /* - * @param c - * @param currentfs We return this if we did not make a new one. - * @param uniqueName Same name used to help identify the created fs. - * @return A new fs instance if we are up on DistributeFileSystem. - * @throws IOException - */ - private static Configuration setDifferentUser(final Configuration c) - throws IOException { - FileSystem currentfs = FileSystem.get(c); - if (!(currentfs instanceof DistributedFileSystem)) return c; - // Else distributed filesystem. Make a new instance per daemon. Below - // code is taken from the AppendTestUtil over in hdfs. - Configuration c2 = new Configuration(c); - String username = UGI.getUserName() + ".hrs." + index++; - UnixUserGroupInformation.saveToConf(c2, - UnixUserGroupInformation.UGI_PROPERTY_NAME, - new UnixUserGroupInformation(username, new String[]{"supergroup"})); - return c2; + super(HBaseTestingUtility.setDifferentUser(conf, index++)); } @Override diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java index acb4fdc..94aff3d 100644 --- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java @@ -31,13 +31,13 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue.KeyComparator; import org.apache.hadoop.hbase.io.hfile.HFile.BlockIndex; import org.apache.hadoop.hbase.io.hfile.HFile.Reader; import org.apache.hadoop.hbase.io.hfile.HFile.Writer; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; -import org.apache.hadoop.io.RawComparator; import org.apache.hadoop.io.Writable; /** @@ -52,7 +52,7 @@ public class TestHFile extends HBaseTestCase { static final Log LOG = LogFactory.getLog(TestHFile.class); private static String ROOT_DIR = - System.getProperty("test.build.data", "/tmp/TestHFile"); + HBaseTestingUtility.getTestDir("TestHFile").toString(); private final int minBlockSize = 512; private static String localFormatter = "%010d"; diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java index 7edfa34..d99fc1c 100644 --- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.compress.CompressionCodec; @@ -45,7 +46,7 @@ import org.apache.hadoop.io.compress.GzipCodec; */ public class TestHFilePerformance extends TestCase { private static String ROOT_DIR = - System.getProperty("test.build.data", "/tmp/TestHFilePerformance"); + HBaseTestingUtility.getTestDir("TestHFilePerformance").toString(); private FileSystem fs; private Configuration conf; private long startTimeEpoch; diff --git a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java index de6fd79..307e642 100644 --- a/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java +++ b/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.io.hfile.HFile.Reader; import org.apache.hadoop.hbase.io.hfile.HFile.Writer; import org.apache.hadoop.io.BytesWritable; @@ -235,7 +236,7 @@ public class TestHFileSeek extends TestCase { int maxWordLen = 20; String rootDir = - System.getProperty("test.build.data", "/tmp/TestTFileSeek"); + HBaseTestingUtility.getTestDir("TestTFileSeek").toString(); String file = "TestTFileSeek"; // String compress = "lzo"; DISABLED String compress = "none"; diff --git a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java index dcda84c..fd77329 100644 --- a/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java +++ b/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java @@ -27,8 +27,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.io.Reference.Range; @@ -308,7 +308,7 @@ public class TestStoreFile extends HBaseTestCase { } private static String ROOT_DIR = - System.getProperty("test.build.data", "/tmp/TestStoreFile"); + HBaseTestingUtility.getTestDir("TestStoreFile").toString(); private static String localFormatter = "%010d"; public void testBloomFilter() throws Exception {