diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java index dbff761d27..fd91b29c90 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java @@ -243,7 +243,7 @@ private static void loopUntilHMSReady(String msHost, int port, AtomicBoolean hou LOG.info("HMS started, waiting for housekeeper threads to start."); } } catch (Exception e) { - LOG.info("Waiting the HMS to start."); + LOG.info("Waiting the HMS to start on port {}.", port); exc = e; } Thread.sleep(1000); diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/MiniHMS.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/MiniHMS.java index 0b2e4d1376..96f9327f60 100644 --- standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/MiniHMS.java +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/MiniHMS.java @@ -18,8 +18,18 @@ package org.apache.hadoop.hive.metastore.minihms; +import java.io.File; +import java.net.URL; + import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; +import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Mini HMS implementation, which can be used to run tests against different HMS configurations. @@ -29,7 +39,10 @@ * through the Thrift interface * - CLUSTER - In this case the MiniHMS is only a wrapper around the HMS running on a cluster, * so the same tests could be run against a real cluster + * - STANDALONE - In this case the MiniHMS starts up and listens for Thrift requests on a + * random port. */ +@Category(MetastoreUnitTest.class) public class MiniHMS { /** * The possible MetaStore types. @@ -37,7 +50,8 @@ public enum MiniHMSType { EMBEDDED, REMOTE, - CLUSTER + CLUSTER, + STANDALONE } /** @@ -68,9 +82,46 @@ public AbstractMetaStoreService build() throws Exception { return new EmbeddedMetaStoreForTests(metaStoreConf); case CLUSTER: return new ClusterMetaStoreForTests(metaStoreConf); + case STANDALONE: + return new StandaloneMetaStoreForTests(metaStoreConf); default: throw new IllegalArgumentException("Unexpected miniHMSType: " + miniHMSType); } } } + + @Test + public void testRunCluster() throws Exception { + if (!Boolean.parseBoolean(System.getProperty("miniHMS.run", "false"))) { + return; + } + + Logger LOG = LoggerFactory.getLogger(this.getClass()); + + MiniHMSType clusterType = MiniHMSType.STANDALONE; + String confFilesProperty = + System.getProperty("miniHMS.conf", "./src/main/resources/metastore-site.xml"); + + Integer minutes = Integer.valueOf(System.getProperty("miniHMS.minutes", "30")); + + Configuration conf = new Configuration(false); + + // Load conf files + String[] confFiles = confFilesProperty.split(","); + int idx = 0; + for (; idx < confFiles.length; ++idx) { + String confFile = confFiles[idx]; + if (confFile.isEmpty()) { + continue; + } + conf.addResource(new URL("file://" + new File(confFile).toURI().getPath())); + } + conf.set("datanucleus.schema.autoCreateAll", "true"); + AbstractMetaStoreService miniHMS = + new MiniHMS.Builder().setConf(conf).setType(clusterType).build(); + miniHMS.start(); + int port = MetastoreConf.getIntVar(miniHMS.getConfiguration(), ConfVars.SERVER_PORT); + LOG.warn(String.format("Started HMS on thrift://localhost:%d for %d minutes", port, minutes)); + java.util.concurrent.TimeUnit.MINUTES.sleep(minutes); + } } diff --git standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/StandaloneMetaStoreForTests.java standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/StandaloneMetaStoreForTests.java new file mode 100644 index 0000000000..1fc68197a4 --- /dev/null +++ standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/StandaloneMetaStoreForTests.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.minihms; + +import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CONNECT_URL_KEY; +import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID; +import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.EXECUTE_SET_UGI; +import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.SCHEMA_VERIFICATION; +import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.SERVER_PORT; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; + +/** + * The AbstractMetaStore implementation which is used for standalone testing. This opens a random + * port and starts up an HMS instance for accessing via Thrift. + */ + +public class StandaloneMetaStoreForTests extends AbstractMetaStoreService { + public StandaloneMetaStoreForTests(Configuration configuration) { + super(configuration); + } + + public void start() throws Exception { + MetastoreConf.setBoolVar(getConfiguration(), EXECUTE_SET_UGI, false); + + boolean hasJdbcUri = getConfiguration().get(CONNECT_URL_KEY.getVarname()) != null; + boolean hasAcidOn = getConfiguration().get(CREATE_TABLES_AS_ACID.getVarname()) != null; + + if (!hasJdbcUri) { + MetastoreConf.setBoolVar(getConfiguration(), SCHEMA_VERIFICATION, false); + } + + Path wh = Files.createTempDirectory("hms", + PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------"))); + + MetastoreConf.setVar(getConfiguration(), ConfVars.WAREHOUSE, wh.toUri().toString()); + + int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), + getConfiguration(), hasJdbcUri, false, hasAcidOn, true, hasAcidOn); + + MetastoreConf.setLongVar(getConfiguration(), SERVER_PORT, port); + super.start(); + } +}