diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java index 70cbf04823..0c07d2c139 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreHBaseCliDriver.java @@ -34,7 +34,6 @@ public class CoreHBaseCliDriver extends CliAdapter { private HBaseQTestUtil qt; - private HBaseTestSetup setup = new HBaseTestSetup(); public CoreHBaseCliDriver(AbstractCliConfig testCliConfig) { super(testCliConfig); @@ -49,7 +48,7 @@ public void beforeClass() { try { qt = new HBaseQTestUtil(cliConfig.getResultsDir(), cliConfig.getLogDir(), miniMR, - setup, initScript, cleanupScript); + new HBaseTestSetup(), initScript, cleanupScript); qt.newSession(); qt.cleanUp(null); qt.createSources(null); @@ -94,7 +93,6 @@ public void tearDown() { public void shutdown() throws Exception { try { qt.shutdown(); - setup.tearDown(); } catch (Exception e) { System.err.println("Exception: " + e.getMessage()); e.printStackTrace(); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java index 55e744e0f3..e100f58772 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java @@ -65,7 +65,7 @@ public void beforeClass() { String hadoopVer = cliConfig.getHadoopVersion(); qt = new QTestUtil(cliConfig.getResultsDir(), cliConfig.getLogDir(), miniMR, hiveConfDir, hadoopVer, initScript, - cleanupScript, false, null); + cleanupScript, false); // do a one time initialization qt.newSession(); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java index 07df0c9d1e..234bbff36e 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java @@ -20,7 +20,6 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.ql.QTestUtil; /** @@ -37,29 +36,14 @@ /** A handle to this harness's cluster */ private final Connection conn; - private HBaseTestSetup hbaseSetup = null; - public HBaseQTestUtil( String outDir, String logDir, MiniClusterType miniMr, HBaseTestSetup setup, String initScript, String cleanupScript) throws Exception { - super(outDir, logDir, miniMr, null, "0.20", initScript, cleanupScript, false); - hbaseSetup = setup; - hbaseSetup.preTest(conf); + super(outDir, logDir, miniMr, null, "0.20", initScript, cleanupScript, + false, setup); this.conn = setup.getConnection(); - super.init(); - } - - @Override - public void init() throws Exception { - // defer - } - - @Override - protected void initConfFromSetup() throws Exception { - super.initConfFromSetup(); - hbaseSetup.preTest(conf); } @Override @@ -95,9 +79,4 @@ public void cleanUp(String tname) throws Exception { } } - @Override - public void clearTestSideEffects() throws Exception { - super.clearTestSideEffects(); - hbaseSetup.preTest(conf); - } } diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java index 7b203a9281..52c37ef78d 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java @@ -19,23 +19,25 @@ package org.apache.hadoop.hive.hbase; import java.io.IOException; -import java.net.ServerSocket; import java.util.Arrays; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.MiniHBaseCluster; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hadoop.mapred.JobConf; import org.apache.zookeeper.Watcher; @@ -43,11 +45,11 @@ * HBaseTestSetup defines HBase-specific test fixtures which are * reused across testcases. */ -public class HBaseTestSetup { +public class HBaseTestSetup extends QTestUtil.QTestSetup { private MiniHBaseCluster hbaseCluster; + private HBaseTestingUtility util; private int zooKeeperPort; - private String hbaseRoot; private Connection hbaseConn; private static final int NUM_REGIONSERVERS = 1; @@ -56,13 +58,16 @@ public Connection getConnection() { return this.hbaseConn; } - void preTest(HiveConf conf) throws Exception { + @Override + public void preTest(HiveConf conf) throws Exception { + super.preTest(conf); setUpFixtures(conf); - conf.set("hbase.rootdir", hbaseRoot); - conf.set("hbase.master", hbaseCluster.getMaster().getServerName().getHostAndPort()); - conf.set("hbase.zookeeper.property.clientPort", Integer.toString(zooKeeperPort)); + // Set some properties since HiveConf gets recreated for the new query + Path hbaseRoot = util.getDefaultRootDirPath(); + conf.set(HConstants.HBASE_DIR, hbaseRoot.toUri().toString()); + String auxJars = conf.getAuxJars(); auxJars = (StringUtils.isBlank(auxJars) ? "" : (auxJars + ",")) + "file://" + new JobConf(conf, HBaseConfiguration.class).getJar(); @@ -76,30 +81,23 @@ private void setUpFixtures(HiveConf conf) throws Exception { * QTestUtil already starts it. */ int zkPort = conf.getInt("hive.zookeeper.client.port", -1); + conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(zkPort)); if ((zkPort == zooKeeperPort) && (hbaseCluster != null)) { return; } zooKeeperPort = zkPort; - String tmpdir = System.getProperty("test.tmp.dir"); this.tearDown(); - conf.set("hbase.master", "local"); - - hbaseRoot = "file:///" + tmpdir + "/hbase"; - conf.set("hbase.rootdir", hbaseRoot); - conf.set("hbase.zookeeper.property.clientPort", - Integer.toString(zooKeeperPort)); - Configuration hbaseConf = HBaseConfiguration.create(conf); - hbaseConf.setInt("hbase.master.port", findFreePort()); - hbaseConf.setInt("hbase.master.info.port", -1); - hbaseConf.setInt("hbase.regionserver.port", findFreePort()); - hbaseConf.setInt("hbase.regionserver.info.port", -1); // Fix needed due to dependency for hbase-mapreduce module + // Check CDH-59433 for details System.setProperty("org.apache.hadoop.hbase.shaded.io.netty.packagePrefix", "org.apache.hadoop.hbase.shaded."); - hbaseCluster = new MiniHBaseCluster(hbaseConf, NUM_REGIONSERVERS); - conf.set("hbase.master", hbaseCluster.getMaster().getServerName().getHostAndPort()); - hbaseConn = ConnectionFactory.createConnection(hbaseConf); + + Configuration hbaseConf = HBaseConfiguration.create(conf); + util = new HBaseTestingUtility(hbaseConf); + util.startMiniDFSCluster(1); + hbaseCluster = util.startMiniHBaseCluster(1, NUM_REGIONSERVERS); + hbaseConn = util.getConnection(); // opening the META table ensures that cluster is running Table meta = null; @@ -160,20 +158,10 @@ private void createHBaseTable() throws IOException { } } - private static int findFreePort() throws IOException { - ServerSocket server = new ServerSocket(0); - int port = server.getLocalPort(); - server.close(); - return port; - } - + @Override public void tearDown() throws Exception { - if (hbaseConn != null) { - hbaseConn.close(); - hbaseConn = null; - } if (hbaseCluster != null) { - hbaseCluster.shutdown(); + util.shutdownMiniCluster(); hbaseCluster = null; } } diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java index 5adbb63693..c62cf96e90 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java @@ -488,12 +488,27 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadoopVer, String initScript, String cleanupScript, boolean withLlapIo) throws Exception { this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, - withLlapIo, null); + withLlapIo, new QTestSetup()); + } + + public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, + String confDir, String hadoopVer, String initScript, String cleanupScript, + boolean withLlapIo, QTestSetup qtestSetup) throws Exception { + this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, + withLlapIo, null, qtestSetup); + } + + public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, + String confDir, String hadoopVer, String initScript, String cleanupScript, + boolean withLlapIo, FsType fsType) + throws Exception { + this(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, + withLlapIo, fsType, new QTestSetup()); } public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String confDir, String hadoopVer, String initScript, String cleanupScript, - boolean withLlapIo, FsType fsType) + boolean withLlapIo, FsType fsType, QTestSetup qtestSetup) throws Exception { LOG.info("Setting up QTestUtil with outDir={}, logDir={}, clusterType={}, confDir={}," + " hadoopVer={}, initScript={}, cleanupScript={}, withLlapIo={}," + @@ -536,7 +551,7 @@ public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, setupFileSystem(shims); - setup = new QTestSetup(); + setup = qtestSetup; setup.preTest(conf); setupMiniCluster(shims, confDir);