diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java index 9fa263d53596378276fabc43b7f44409b4b0ccfb..6ff48eeadad22ad983a6c248bc522758d23a6f47 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java @@ -33,8 +33,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStore; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.SerDeInfo; @@ -89,7 +89,7 @@ private static HiveConf hiveConf; private static File workDir; - private static final String msPort = "20199"; + private static int msPort; private static Thread t; static { @@ -98,22 +98,6 @@ schemaMap.put(tableNames[2], new HCatSchema(ColumnHolder.hCattest3Cols)); } - private static class RunMS implements Runnable { - - @Override - public void run() { - try { - String warehouseConf = HiveConf.ConfVars.METASTOREWAREHOUSE.varname + "=" - + warehousedir.toString(); - HiveMetaStore.main(new String[]{"-v", "-p", msPort, "--hiveconf", warehouseConf}); - } catch (Throwable t) { - System.err.println("Exiting. Got exception from metastore: " + t.getMessage()); - t.printStackTrace(); - } - } - - } - /** * Private class which holds all the data for the test cases */ @@ -173,10 +157,11 @@ public static void setup() throws Exception { warehousedir = new Path(System.getProperty("test.warehouse.dir")); - // Run hive metastore server - t = new Thread(new RunMS()); - t.start(); + HiveConf metastoreConf = new HiveConf(); + metastoreConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, warehousedir.toString()); + // Run hive metastore server + msPort = MetaStoreUtils.startMetaStore(metastoreConf); // LocalJobRunner does not work with mapreduce OutputCommitter. So need // to use MiniMRCluster. MAPREDUCE-2350 Configuration conf = new Configuration(true); diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java index aa9c7d31f2cbe187e84bfddfe411da1f49943819..48ee7cf1e1c252e60c3f21154e972030c3126b50 100644 --- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java +++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java @@ -34,8 +34,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStore; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NotificationEvent; @@ -83,41 +83,14 @@ public class TestHCatClient { private static final Logger LOG = LoggerFactory.getLogger(TestHCatClient.class); - private static final String msPort = "20101"; + private static int msPort; private static HiveConf hcatConf; private static boolean isReplicationTargetHCatRunning = false; - private static final String replicationTargetHCatPort = "20102"; + private static int replicationTargetHCatPort; private static HiveConf replicationTargetHCatConf; private static SecurityManager securityManager; private static boolean useExternalMS = false; - public static class RunMS implements Runnable { - - private final String msPort; - private List args = new ArrayList(); - - public RunMS(String msPort) { - this.msPort = msPort; - this.args.add("-v"); - this.args.add("-p"); - this.args.add(this.msPort); - } - - public RunMS arg(String arg) { - this.args.add(arg); - return this; - } - - @Override - public void run() { - try { - HiveMetaStore.main(args.toArray(new String[args.size()])); - } catch (Throwable t) { - LOG.error("Exiting. Got exception from metastore: ", t); - } - } - } // class RunMS; - @AfterClass public static void tearDown() throws Exception { if (!useExternalMS) { @@ -142,10 +115,7 @@ public static void startMetaStoreServer() throws Exception { System.setProperty(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname, DbNotificationListener.class.getName()); // turn on db notification listener on metastore - Thread t = new Thread(new RunMS(msPort)); - t.start(); - Thread.sleep(10000); - + msPort = MetaStoreUtils.startMetaStore(); securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" @@ -832,13 +802,10 @@ public void testDropPartitionsWithPartialSpec() throws Exception { private void startReplicationTargetMetaStoreIfRequired() throws Exception { if (!isReplicationTargetHCatRunning) { - Thread t = new Thread(new RunMS(replicationTargetHCatPort) - .arg("--hiveconf") - .arg("javax.jdo.option.ConnectionURL") // Reset, to use a different Derby instance. - .arg(hcatConf.get("javax.jdo.option.ConnectionURL") - .replace("metastore", "target_metastore"))); - t.start(); - Thread.sleep(10000); + HiveConf conf = new HiveConf(); + conf.set("javax.jdo.option.ConnectionURL", hcatConf.get("javax.jdo.option.ConnectionURL") + .replace("metastore", "target_metastore")); + replicationTargetHCatPort = MetaStoreUtils.startMetaStore(conf); replicationTargetHCatConf = new HiveConf(hcatConf); replicationTargetHCatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + replicationTargetHCatPort); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java index 7576f3923c7ae2e144263445d0a69e5424489567..239bad05ff335274f2fcbbbc910581f8ccf61659 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java @@ -19,37 +19,13 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; public class TestMarkPartitionRemote extends TestMarkPartition { - private static class RunMS implements Runnable { - - private final int port; - - public RunMS(int port) { - this.port = port; - } - @Override - public void run() { - try { - HiveMetaStore.main(new String[] { String.valueOf(port) }); - } catch (Throwable e) { - e.printStackTrace(System.err); - assert false; - } - } - - } @Override protected void setUp() throws Exception { super.setUp(); - int port = MetaStoreUtils.findFreePort(); - Thread t = new Thread(new RunMS(port)); - t.setDaemon(true); - t.start(); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + MetaStoreUtils.startMetaStore()); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); - Thread.sleep(30000); } } diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index da3da8b209ad9d390fef5fb8c585422e23a9ff0f..e01fe45cb5994c3bf7b96a83ddd143d41015012c 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -1176,22 +1176,38 @@ public static void makeDir(Path path, HiveConf hiveConf) throws MetaException { } catch (IOException e) { throw new MetaException("Unable to : " + path); } + } + public static int startMetaStore() throws Exception { + return startMetaStore(ShimLoader.getHadoopThriftAuthBridge(), null); } - public static void startMetaStore(final int port, - final HadoopThriftAuthBridge bridge) throws Exception { - startMetaStore(port, bridge, new HiveConf(HMSHandler.class)); + public static int startMetaStore(final HadoopThriftAuthBridge bridge, HiveConf conf) throws Exception { + int port = findFreePort(); + startMetaStore(port, bridge, conf); + return port; + } + + public static int startMetaStore(HiveConf conf) throws Exception { + return startMetaStore(ShimLoader.getHadoopThriftAuthBridge(), conf); + } + + public static void startMetaStore(final int port, final HadoopThriftAuthBridge bridge) throws Exception { + startMetaStore(port, bridge, null); } public static void startMetaStore(final int port, - final HadoopThriftAuthBridge bridge, final HiveConf hiveConf) + final HadoopThriftAuthBridge bridge, HiveConf hiveConf) throws Exception{ + if (hiveConf == null) { + hiveConf = new HiveConf(HMSHandler.class); + } + final HiveConf finalHiveConf = hiveConf; Thread thread = new Thread(new Runnable() { @Override public void run() { try { - HiveMetaStore.startMetaStore(port, bridge, hiveConf); + HiveMetaStore.startMetaStore(port, bridge, finalHiveConf); } catch (Throwable e) { LOG.error("Metastore Thrift Server threw an exception...",e); } diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java index 3f4561cc4b2a737f18e274779d6e748dcd3abad0..da883d20482a433c3803b70a85e0134cf4c77b44 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java @@ -40,7 +40,6 @@ public ExpectedException thrown = ExpectedException.none(); private static final Logger LOG = LoggerFactory.getLogger(TestHiveMetaStoreGetMetaConf.class); - private static final String msPort = "20103"; private static HiveConf hiveConf; private static SecurityManager securityManager; @@ -65,20 +64,6 @@ public void checkExit(int status) { } } - private static class RunMS implements Runnable { - - @Override - public void run() { - try { - HiveMetaStore.main(new String[]{"-v", "-p", msPort, "--hiveconf", - "hive.metastore.expression.proxy=" + MockPartitionExpressionForMetastore.class.getCanonicalName(), - "--hiveconf", "hive.metastore.try.direct.sql.ddl=false"}); - } catch (Throwable t) { - LOG.error("Exiting. Got exception from metastore: ", t); - } - } - } - @AfterClass public static void tearDown() throws Exception { LOG.info("Shutting down metastore."); @@ -90,7 +75,11 @@ public static void startMetaStoreServer() throws Exception { securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); - + HiveConf metastoreConf = new HiveConf(); + metastoreConf.setClass(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS.varname, + MockPartitionExpressionForMetastore.class, PartitionExpressionProxy.class); + metastoreConf.setBoolVar(HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL_DDL, false); + int msPort = MetaStoreUtils.startMetaStore(metastoreConf); hiveConf = new HiveConf(TestHiveMetaStoreGetMetaConf.class); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + msPort); @@ -101,8 +90,6 @@ public static void startMetaStoreServer() throws Exception { System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " "); - - new Thread(new RunMS()).start(); } @Before diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java index 922a4bf11cbc5693cc5753b8488c5ae3085dd3f4..c1646c3c157b145f19fb4bba9b3dbe9a1c1db75d 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java @@ -49,7 +49,7 @@ public class TestHiveMetaStorePartitionSpecs { private static final Logger LOG = LoggerFactory.getLogger(TestHiveMetaStorePartitionSpecs.class); - private static final String msPort = "20102"; + private static int msPort; private static HiveConf hiveConf; private static SecurityManager securityManager; @@ -73,18 +73,6 @@ public void checkExit(int status) { } } - private static class RunMS implements Runnable { - - @Override - public void run() { - try { - HiveMetaStore.main(new String[]{"-v", "-p", msPort, "--hiveconf", - "hive.metastore.expression.proxy=" + MockPartitionExpressionForMetastore.class.getCanonicalName()}); - } catch (Throwable t) { - LOG.error("Exiting. Got exception from metastore: ", t); - } - } - } @AfterClass public static void tearDown() throws Exception { @@ -95,10 +83,10 @@ public static void tearDown() throws Exception { @BeforeClass public static void startMetaStoreServer() throws Exception { - Thread t = new Thread(new RunMS()); - t.start(); - Thread.sleep(10000); - + HiveConf metastoreConf = new HiveConf(); + metastoreConf.setClass(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS.varname, + MockPartitionExpressionForMetastore.class, PartitionExpressionProxy.class); + msPort = MetaStoreUtils.startMetaStore(metastoreConf); securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); hiveConf = new HiveConf(TestHiveMetaStorePartitionSpecs.class); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java index ee95e160c858a93c43f96775b360f28f8d6a178f..9dfcd54b3e8e4e6b3bde0a6c0f58f077962849e0 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java @@ -22,7 +22,7 @@ import java.net.ServerSocket; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.util.StringUtils; /** @@ -34,43 +34,13 @@ public class TestHiveRemote extends TestHive { /** - * Starts a remote metastore - */ - private static class RunMS implements Runnable { - String port; - - public RunMS(String port) { - this.port = port; - } - - @Override - public void run() { - try { - HiveMetaStore.main(new String[] { port }); - } catch (Throwable e) { - e.printStackTrace(System.err); - assert false; - } - } - - } - - /** * Start a remote metastore and initialize a Hive object pointing at it. */ @Override protected void setUp() throws Exception { super.setUp(); hiveConf = new HiveConf(this.getClass()); - String port = findFreePort(); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); - - Thread t = new Thread(new RunMS(port)); - t.start(); - - // Wait a little bit for the metastore to start. - Thread.sleep(5000); - + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + MetaStoreUtils.startMetaStore()); try { hm = Hive.get(hiveConf);