diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java index 8a2151c..3c9d89a 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.Type; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.io.HiveInputFormat; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -62,7 +63,6 @@ public class TestPermsGrp extends TestCase { private boolean isServerRunning = false; - private int msPort; private HiveConf hcatConf; private Warehouse clientWH; private HiveMetaStoreClient msc; @@ -80,7 +80,8 @@ protected void setUp() throws Exception { return; } - msPort = MetaStoreTestUtils.startMetaStoreWithRetry(); + hcatConf = new HiveConf(this.getClass()); + MetaStoreTestUtils.startMetaStoreWithRetry(hcatConf); isServerRunning = true; @@ -88,8 +89,6 @@ protected void setUp() throws Exception { System.setSecurityManager(new NoExitSecurityManager()); Policy.setPolicy(new DerbyPolicy()); - hcatConf = new HiveConf(this.getClass()); - hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://127.0.0.1:" + msPort); hcatConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); hcatConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES, 3); @@ -102,6 +101,12 @@ protected void setUp() throws Exception { msc = new HiveMetaStoreClient(hcatConf); System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " "); + System.setProperty(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, + MetastoreConf.getVar(hcatConf, MetastoreConf.ConfVars.WAREHOUSE)); + System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, + MetastoreConf.getVar(hcatConf, MetastoreConf.ConfVars.CONNECT_URL_KEY)); + System.setProperty(HiveConf.ConfVars.METASTOREURIS.varname, + MetastoreConf.getVar(hcatConf, MetastoreConf.ConfVars.THRIFT_URIS)); } public void testCustomPerms() throws Exception { diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java index d9de10e..8a8a326 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java @@ -90,9 +90,6 @@ private static HiveConf hiveConf; private static File workDir; - private static int msPort; - private static Thread t; - static { schemaMap.put(tableNames[0], new HCatSchema(ColumnHolder.hCattest1Cols)); schemaMap.put(tableNames[1], new HCatSchema(ColumnHolder.hCattest2Cols)); @@ -162,7 +159,7 @@ public static void setup() throws Exception { metastoreConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, warehousedir.toString()); // Run hive metastore server - msPort = MetaStoreTestUtils.startMetaStoreWithRetry(metastoreConf); + MetaStoreTestUtils.startMetaStoreWithRetry(metastoreConf); // Read the warehouse dir, which can be changed so multiple MetaStore tests could be run on // the same server warehousedir = new Path(MetastoreConf.getVar(metastoreConf, MetastoreConf.ConfVars.WAREHOUSE)); @@ -178,15 +175,14 @@ public static void setup() throws Exception { new JobConf(conf)); mrConf = mrCluster.createJobConf(); - initializeSetup(); + initializeSetup(metastoreConf); warehousedir.getFileSystem(conf).mkdirs(warehousedir); } - private static void initializeSetup() throws Exception { + private static void initializeSetup(HiveConf metastoreConf) throws Exception { - hiveConf = new HiveConf(mrConf, TestHCatMultiOutputFormat.class); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + msPort); + hiveConf = new HiveConf(metastoreConf, TestHCatMultiOutputFormat.class); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES, 3); hiveConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, @@ -196,6 +192,12 @@ private static void initializeSetup() throws Exception { hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " "); + System.setProperty(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, + MetastoreConf.getVar(hiveConf, MetastoreConf.ConfVars.WAREHOUSE)); + System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, + MetastoreConf.getVar(hiveConf, MetastoreConf.ConfVars.CONNECT_URL_KEY)); + System.setProperty(HiveConf.ConfVars.METASTOREURIS.varname, + MetastoreConf.getVar(hiveConf, MetastoreConf.ConfVars.THRIFT_URIS)); hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, warehousedir.toString()); try { diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java index fb6a7f4..cd7dd83 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; @@ -73,7 +74,6 @@ private static FileSystem fs = null; private static MiniMRCluster mrCluster = null; private static boolean isServerRunning = false; - private static int msPort; private static HiveConf hcatConf; private static HiveMetaStoreClient msc; private static SecurityManager securityManager; @@ -106,17 +106,14 @@ public static void setup() throws Exception { return; } - msPort = MetaStoreTestUtils.startMetaStoreWithRetry(); + hcatConf = new HiveConf(TestHCatPartitionPublish.class); + MetaStoreTestUtils.startMetaStoreWithRetry(hcatConf); - Thread.sleep(10000); isServerRunning = true; securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); Policy.setPolicy(new DerbyPolicy()); - hcatConf = new HiveConf(TestHCatPartitionPublish.class); - hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" - + msPort); hcatConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); hcatConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES, 3); hcatConf.setTimeVar(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT, 120, TimeUnit.SECONDS); @@ -129,6 +126,12 @@ public static void setup() throws Exception { msc = new HiveMetaStoreClient(hcatConf); System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " "); + System.setProperty(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, + MetastoreConf.getVar(hcatConf, MetastoreConf.ConfVars.WAREHOUSE)); + System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, + MetastoreConf.getVar(hcatConf, MetastoreConf.ConfVars.CONNECT_URL_KEY)); + System.setProperty(HiveConf.ConfVars.METASTOREURIS.varname, + MetastoreConf.getVar(hcatConf, MetastoreConf.ConfVars.THRIFT_URIS)); } @AfterClass diff --git itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index 23fcbe8..681d722 100644 --- itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hive.llap.LlapItUtils; import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper; import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim; @@ -342,6 +343,7 @@ public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType, boolean usePortsF public void start(Map confOverlay) throws Exception { if (isMetastoreRemote) { MetaStoreTestUtils.startMetaStoreWithRetry(getHiveConf()); + setWareHouseDir(MetastoreConf.getVar(getHiveConf(), MetastoreConf.ConfVars.WAREHOUSE)); } // Set confOverlay parameters diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index f007261..7f4b79a 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -1097,8 +1097,10 @@ public static void setHiveSiteLocation(URL location) { } public static Configuration newMetastoreConf() { + return newMetastoreConf(new Configuration()); + } - Configuration conf = new Configuration(); + public static Configuration newMetastoreConf(Configuration conf) { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); if (classLoader == null) { diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java index 1d12cf9..c7dafe3 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java @@ -27,11 +27,15 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.events.EventCleanerTask; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; +import org.apache.hadoop.hive.metastore.utils.FileUtils; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -101,15 +105,36 @@ public static int startMetaStoreWithRetry(HadoopThriftAuthBridge bridge, Configuration conf) throws Exception { Exception metaStoreException = null; String warehouseDir = MetastoreConf.getVar(conf, ConfVars.WAREHOUSE); + String jdbcUrl = MetastoreConf.getVar(conf, ConfVars.CONNECT_URL_KEY); + for (int tryCount = 0; tryCount < MetaStoreTestUtils.RETRY_COUNT; tryCount++) { try { int metaStorePort = findFreePort(); + // Setting metastore instance specific warehouse directory, postfixing with port Path postfixedWarehouseDir = new Path(warehouseDir, String.valueOf(metaStorePort)); MetastoreConf.setVar(conf, ConfVars.WAREHOUSE, postfixedWarehouseDir.toString()); + + // Setting metastore instance specific jdbc url postfixed with port + String postfixedJdbcUrl = jdbcUrl.replace("junit_metastore_db", "junit_metastore_db_" + + metaStorePort); + MetastoreConf.setVar(conf, ConfVars.CONNECT_URL_KEY, postfixedJdbcUrl); + + // Setting metastore instance specific metastore uri MetastoreConf.setVar(conf, ConfVars.THRIFT_URIS, "thrift://localhost:" + metaStorePort); MetaStoreTestUtils.startMetaStore(metaStorePort, bridge, conf); - LOG.error("MetaStore Thrift Server started on port: {} with warehouse dir: {}", - metaStorePort, postfixedWarehouseDir); + + // Creating warehouse dir, if not exists + Warehouse wh = new Warehouse(conf); + if (!wh.isDir(wh.getWhRoot())) { + FileSystem fs = wh.getWhRoot().getFileSystem(conf); + fs.mkdirs(wh.getWhRoot()); + fs.setPermission(wh.getWhRoot(), + new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); + LOG.info("MetaStore warehouse root dir ({}) is created", postfixedWarehouseDir); + } + + LOG.info("MetaStore Thrift Server started on port: {} with warehouse dir: {} with " + + "jdbcUrl: {}", metaStorePort, postfixedWarehouseDir, postfixedJdbcUrl); return metaStorePort; } catch (ConnectException ce) { metaStoreException = ce; diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 6f52a52..1912730 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -100,7 +100,7 @@ public abstract class TestHiveMetaStore { private static final Logger LOG = LoggerFactory.getLogger(TestHiveMetaStore.class); protected static HiveMetaStoreClient client; - protected static Configuration conf; + protected static Configuration conf = MetastoreConf.newMetastoreConf(); protected static Warehouse warehouse; protected static boolean isThriftClient = false; @@ -113,7 +113,6 @@ @Before public void setUp() throws Exception { - conf = MetastoreConf.newMetastoreConf(); warehouse = new Warehouse(conf); // set some values to use for getting conf. vars @@ -2928,7 +2927,7 @@ private void createFunction(String dbName, String funcName, String className, @Test public void testRetriableClientWithConnLifetime() throws Exception { - Configuration conf = MetastoreConf.newMetastoreConf(); + Configuration conf = MetastoreConf.newMetastoreConf(new Configuration(this.conf)); MetastoreConf.setTimeVar(conf, ConfVars.CLIENT_SOCKET_LIFETIME, 4, TimeUnit.SECONDS); MetaStoreTestUtils.setConfForStandloneMode(conf); long timeout = 5 * 1000; // Lets use a timeout more than the socket lifetime to simulate a reconnect diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java index e1f81bb..415988d 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java @@ -43,7 +43,6 @@ public void setUp() throws Exception { if (isServerStarted) { Assert.assertNotNull("Unable to connect to the MetaStore server", client); - MetastoreConf.setVar(conf, ConfVars.THRIFT_URIS, "thrift://localhost:" + port); return; }