diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java index 9fa263d53596378276fabc43b7f44409b4b0ccfb..17ceeb6c5ccd1c726489af8d811557dacaf4c591 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.net.Socket; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -35,6 +36,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStore; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.SerDeInfo; @@ -89,7 +91,7 @@ private static HiveConf hiveConf; private static File workDir; - private static final String msPort = "20199"; + private static int msPort; private static Thread t; static { @@ -98,22 +100,6 @@ schemaMap.put(tableNames[2], new HCatSchema(ColumnHolder.hCattest3Cols)); } - private static class RunMS implements Runnable { - - @Override - public void run() { - try { - String warehouseConf = HiveConf.ConfVars.METASTOREWAREHOUSE.varname + "=" - + warehousedir.toString(); - HiveMetaStore.main(new String[]{"-v", "-p", msPort, "--hiveconf", warehouseConf}); - } catch (Throwable t) { - System.err.println("Exiting. Got exception from metastore: " + t.getMessage()); - t.printStackTrace(); - } - } - - } - /** * Private class which holds all the data for the test cases */ @@ -173,10 +159,11 @@ public static void setup() throws Exception { warehousedir = new Path(System.getProperty("test.warehouse.dir")); - // Run hive metastore server - t = new Thread(new RunMS()); - t.start(); + HiveConf metastoreConf = new HiveConf(); + metastoreConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, warehousedir.toString()); + // Run hive metastore server + msPort = MetaStoreUtils.startMetaStore(metastoreConf); // LocalJobRunner does not work with mapreduce OutputCommitter. So need // to use MiniMRCluster. MAPREDUCE-2350 Configuration conf = new Configuration(true); diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java index aa9c7d31f2cbe187e84bfddfe411da1f49943819..5a93213344c16267d6eddcd15c8cc19ca74c7124 100644 --- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java +++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.math.BigInteger; +import java.net.Socket; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; @@ -36,6 +37,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStore; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NotificationEvent; @@ -83,41 +85,14 @@ public class TestHCatClient { private static final Logger LOG = LoggerFactory.getLogger(TestHCatClient.class); - private static final String msPort = "20101"; + private static int msPort; private static HiveConf hcatConf; private static boolean isReplicationTargetHCatRunning = false; - private static final String replicationTargetHCatPort = "20102"; + private static int replicationTargetHCatPort; private static HiveConf replicationTargetHCatConf; private static SecurityManager securityManager; private static boolean useExternalMS = false; - public static class RunMS implements Runnable { - - private final String msPort; - private List args = new ArrayList(); - - public RunMS(String msPort) { - this.msPort = msPort; - this.args.add("-v"); - this.args.add("-p"); - this.args.add(this.msPort); - } - - public RunMS arg(String arg) { - this.args.add(arg); - return this; - } - - @Override - public void run() { - try { - HiveMetaStore.main(args.toArray(new String[args.size()])); - } catch (Throwable t) { - LOG.error("Exiting. Got exception from metastore: ", t); - } - } - } // class RunMS; - @AfterClass public static void tearDown() throws Exception { if (!useExternalMS) { @@ -142,10 +117,7 @@ public static void startMetaStoreServer() throws Exception { System.setProperty(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname, DbNotificationListener.class.getName()); // turn on db notification listener on metastore - Thread t = new Thread(new RunMS(msPort)); - t.start(); - Thread.sleep(10000); - + msPort = MetaStoreUtils.startMetaStore(); securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" @@ -832,13 +804,10 @@ public void testDropPartitionsWithPartialSpec() throws Exception { private void startReplicationTargetMetaStoreIfRequired() throws Exception { if (!isReplicationTargetHCatRunning) { - Thread t = new Thread(new RunMS(replicationTargetHCatPort) - .arg("--hiveconf") - .arg("javax.jdo.option.ConnectionURL") // Reset, to use a different Derby instance. - .arg(hcatConf.get("javax.jdo.option.ConnectionURL") - .replace("metastore", "target_metastore"))); - t.start(); - Thread.sleep(10000); + HiveConf conf = new HiveConf(); + conf.set("javax.jdo.option.ConnectionURL", hcatConf.get("javax.jdo.option.ConnectionURL") + .replace("metastore", "target_metastore")); + replicationTargetHCatPort = MetaStoreUtils.startMetaStore(conf); replicationTargetHCatConf = new HiveConf(hcatConf); replicationTargetHCatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + replicationTargetHCatPort); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java index 7576f3923c7ae2e144263445d0a69e5424489567..239bad05ff335274f2fcbbbc910581f8ccf61659 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java @@ -19,37 +19,13 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; public class TestMarkPartitionRemote extends TestMarkPartition { - private static class RunMS implements Runnable { - - private final int port; - - public RunMS(int port) { - this.port = port; - } - @Override - public void run() { - try { - HiveMetaStore.main(new String[] { String.valueOf(port) }); - } catch (Throwable e) { - e.printStackTrace(System.err); - assert false; - } - } - - } @Override protected void setUp() throws Exception { super.setUp(); - int port = MetaStoreUtils.findFreePort(); - Thread t = new Thread(new RunMS(port)); - t.setDaemon(true); - t.start(); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + MetaStoreUtils.startMetaStore()); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); - Thread.sleep(30000); } } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 76220f45c079d8d284421c6c7952698207ece868..d2e13f0253e12664ce206bc841f3124e779bb288 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -1169,22 +1169,38 @@ public static void makeDir(Path path, HiveConf hiveConf) throws MetaException { } catch (IOException e) { throw new MetaException("Unable to : " + path); } + } + public static int startMetaStore() throws Exception { + return startMetaStore(ShimLoader.getHadoopThriftAuthBridge(), null); } - public static void startMetaStore(final int port, - final HadoopThriftAuthBridge bridge) throws Exception { - startMetaStore(port, bridge, new HiveConf(HMSHandler.class)); + public static int startMetaStore(final HadoopThriftAuthBridge bridge, HiveConf conf) throws Exception { + int port = findFreePort(); + startMetaStore(port, bridge, conf); + return port; + } + + public static int startMetaStore(HiveConf conf) throws Exception { + return startMetaStore(ShimLoader.getHadoopThriftAuthBridge(), conf); + } + + public static void startMetaStore(final int port, final HadoopThriftAuthBridge bridge) throws Exception { + startMetaStore(port, bridge, null); } public static void startMetaStore(final int port, - final HadoopThriftAuthBridge bridge, final HiveConf hiveConf) + final HadoopThriftAuthBridge bridge, HiveConf hiveConf) throws Exception{ + if (hiveConf == null) { + hiveConf = new HiveConf(HMSHandler.class); + } + final HiveConf finalHiveConf = hiveConf; Thread thread = new Thread(new Runnable() { @Override public void run() { try { - HiveMetaStore.startMetaStore(port, bridge, hiveConf); + HiveMetaStore.startMetaStore(port, bridge, finalHiveConf); } catch (Throwable e) { LOG.error("Metastore Thrift Server threw an exception...",e); } diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java index 3f4561cc4b2a737f18e274779d6e748dcd3abad0..5392d0fb279549e5d113dd218b628fb69107a20b 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java @@ -18,6 +18,9 @@ package org.apache.hadoop.hive.metastore; +import java.io.IOException; +import java.net.Socket; +import java.net.UnknownHostException; import java.security.Permission; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -40,7 +43,6 @@ public ExpectedException thrown = ExpectedException.none(); private static final Logger LOG = LoggerFactory.getLogger(TestHiveMetaStoreGetMetaConf.class); - private static final String msPort = "20103"; private static HiveConf hiveConf; private static SecurityManager securityManager; @@ -65,20 +67,6 @@ public void checkExit(int status) { } } - private static class RunMS implements Runnable { - - @Override - public void run() { - try { - HiveMetaStore.main(new String[]{"-v", "-p", msPort, "--hiveconf", - "hive.metastore.expression.proxy=" + MockPartitionExpressionForMetastore.class.getCanonicalName(), - "--hiveconf", "hive.metastore.try.direct.sql.ddl=false"}); - } catch (Throwable t) { - LOG.error("Exiting. Got exception from metastore: ", t); - } - } - } - @AfterClass public static void tearDown() throws Exception { LOG.info("Shutting down metastore."); @@ -90,7 +78,11 @@ public static void startMetaStoreServer() throws Exception { securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); - + HiveConf metastoreConf = new HiveConf(); + metastoreConf.setClass(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS.varname, + MockPartitionExpressionForMetastore.class, PartitionExpressionProxy.class); + metastoreConf.setBoolVar(HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL_DDL, false); + int msPort = MetaStoreUtils.startMetaStore(metastoreConf); hiveConf = new HiveConf(TestHiveMetaStoreGetMetaConf.class); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + msPort); @@ -101,8 +93,6 @@ public static void startMetaStoreServer() throws Exception { System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " "); - - new Thread(new RunMS()).start(); } @Before diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java index 922a4bf11cbc5693cc5753b8488c5ae3085dd3f4..304c24bc21abf8078344a90a036a8182971cdd79 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java @@ -36,6 +36,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.net.Socket; import java.security.Permission; import java.util.ArrayList; import java.util.Arrays; @@ -49,7 +51,7 @@ public class TestHiveMetaStorePartitionSpecs { private static final Logger LOG = LoggerFactory.getLogger(TestHiveMetaStorePartitionSpecs.class); - private static final String msPort = "20102"; + private static int msPort; private static HiveConf hiveConf; private static SecurityManager securityManager; @@ -73,18 +75,6 @@ public void checkExit(int status) { } } - private static class RunMS implements Runnable { - - @Override - public void run() { - try { - HiveMetaStore.main(new String[]{"-v", "-p", msPort, "--hiveconf", - "hive.metastore.expression.proxy=" + MockPartitionExpressionForMetastore.class.getCanonicalName()}); - } catch (Throwable t) { - LOG.error("Exiting. Got exception from metastore: ", t); - } - } - } @AfterClass public static void tearDown() throws Exception { @@ -95,10 +85,10 @@ public static void tearDown() throws Exception { @BeforeClass public static void startMetaStoreServer() throws Exception { - Thread t = new Thread(new RunMS()); - t.start(); - Thread.sleep(10000); - + HiveConf metastoreConf = new HiveConf(); + metastoreConf.setClass(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS.varname, + MockPartitionExpressionForMetastore.class, PartitionExpressionProxy.class); + msPort = MetaStoreUtils.startMetaStore(metastoreConf); securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); hiveConf = new HiveConf(TestHiveMetaStorePartitionSpecs.class); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java index ee95e160c858a93c43f96775b360f28f8d6a178f..4b2259cd5b949b2a9985c99b58eb63ebbfede825 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java @@ -20,9 +20,11 @@ import java.io.IOException; import java.net.ServerSocket; +import java.net.Socket; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.util.StringUtils; /** @@ -34,43 +36,13 @@ public class TestHiveRemote extends TestHive { /** - * Starts a remote metastore - */ - private static class RunMS implements Runnable { - String port; - - public RunMS(String port) { - this.port = port; - } - - @Override - public void run() { - try { - HiveMetaStore.main(new String[] { port }); - } catch (Throwable e) { - e.printStackTrace(System.err); - assert false; - } - } - - } - - /** * Start a remote metastore and initialize a Hive object pointing at it. */ @Override protected void setUp() throws Exception { super.setUp(); hiveConf = new HiveConf(this.getClass()); - String port = findFreePort(); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); - - Thread t = new Thread(new RunMS(port)); - t.start(); - - // Wait a little bit for the metastore to start. - Thread.sleep(5000); - + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + MetaStoreUtils.startMetaStore()); try { hm = Hive.get(hiveConf);