diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java index 8a2151c..3c9d89a 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.Type; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.io.HiveInputFormat; import org.apache.hadoop.hive.ql.io.HiveOutputFormat; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -62,7 +63,6 @@ public class TestPermsGrp extends TestCase { private boolean isServerRunning = false; - private int msPort; private HiveConf hcatConf; private Warehouse clientWH; private HiveMetaStoreClient msc; @@ -80,7 +80,8 @@ protected void setUp() throws Exception { return; } - msPort = MetaStoreTestUtils.startMetaStoreWithRetry(); + hcatConf = new HiveConf(this.getClass()); + MetaStoreTestUtils.startMetaStoreWithRetry(hcatConf); isServerRunning = true; @@ -88,8 +89,6 @@ protected void setUp() throws Exception { System.setSecurityManager(new NoExitSecurityManager()); Policy.setPolicy(new DerbyPolicy()); - hcatConf = new HiveConf(this.getClass()); - hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://127.0.0.1:" + msPort); hcatConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); hcatConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES, 3); @@ -102,6 +101,12 @@ protected void setUp() throws Exception { msc = new HiveMetaStoreClient(hcatConf); System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " "); + System.setProperty(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, + MetastoreConf.getVar(hcatConf, MetastoreConf.ConfVars.WAREHOUSE)); + System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, + MetastoreConf.getVar(hcatConf, MetastoreConf.ConfVars.CONNECT_URL_KEY)); + System.setProperty(HiveConf.ConfVars.METASTOREURIS.varname, + MetastoreConf.getVar(hcatConf, MetastoreConf.ConfVars.THRIFT_URIS)); } public void testCustomPerms() throws Exception { diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java index d9de10e..8a8a326 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java @@ -90,9 +90,6 @@ private static HiveConf hiveConf; private static File workDir; - private static int msPort; - private static Thread t; - static { schemaMap.put(tableNames[0], new HCatSchema(ColumnHolder.hCattest1Cols)); schemaMap.put(tableNames[1], new HCatSchema(ColumnHolder.hCattest2Cols)); @@ -162,7 +159,7 @@ public static void setup() throws Exception { metastoreConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, warehousedir.toString()); // Run hive metastore server - msPort = MetaStoreTestUtils.startMetaStoreWithRetry(metastoreConf); + MetaStoreTestUtils.startMetaStoreWithRetry(metastoreConf); // Read the warehouse dir, which can be changed so multiple MetaStore tests could be run on // the same server warehousedir = new Path(MetastoreConf.getVar(metastoreConf, MetastoreConf.ConfVars.WAREHOUSE)); @@ -178,15 +175,14 @@ public static void setup() throws Exception { new JobConf(conf)); mrConf = mrCluster.createJobConf(); - initializeSetup(); + initializeSetup(metastoreConf); warehousedir.getFileSystem(conf).mkdirs(warehousedir); } - private static void initializeSetup() throws Exception { + private static void initializeSetup(HiveConf metastoreConf) throws Exception { - hiveConf = new HiveConf(mrConf, TestHCatMultiOutputFormat.class); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + msPort); + hiveConf = new HiveConf(metastoreConf, TestHCatMultiOutputFormat.class); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES, 3); hiveConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, @@ -196,6 +192,12 @@ private static void initializeSetup() throws Exception { hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " "); + System.setProperty(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, + MetastoreConf.getVar(hiveConf, MetastoreConf.ConfVars.WAREHOUSE)); + System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, + MetastoreConf.getVar(hiveConf, MetastoreConf.ConfVars.CONNECT_URL_KEY)); + System.setProperty(HiveConf.ConfVars.METASTOREURIS.varname, + MetastoreConf.getVar(hiveConf, MetastoreConf.ConfVars.THRIFT_URIS)); hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, warehousedir.toString()); try { diff --git hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java index fb6a7f4..cd7dd83 100644 --- hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java +++ hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java @@ -43,6 +43,7 @@ import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; import org.apache.hadoop.hive.ql.io.RCFileOutputFormat; @@ -73,7 +74,6 @@ private static FileSystem fs = null; private static MiniMRCluster mrCluster = null; private static boolean isServerRunning = false; - private static int msPort; private static HiveConf hcatConf; private static HiveMetaStoreClient msc; private static SecurityManager securityManager; @@ -106,17 +106,14 @@ public static void setup() throws Exception { return; } - msPort = MetaStoreTestUtils.startMetaStoreWithRetry(); + hcatConf = new HiveConf(TestHCatPartitionPublish.class); + MetaStoreTestUtils.startMetaStoreWithRetry(hcatConf); - Thread.sleep(10000); isServerRunning = true; securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); Policy.setPolicy(new DerbyPolicy()); - hcatConf = new HiveConf(TestHCatPartitionPublish.class); - hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" - + msPort); hcatConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); hcatConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES, 3); hcatConf.setTimeVar(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT, 120, TimeUnit.SECONDS); @@ -129,6 +126,12 @@ public static void setup() throws Exception { msc = new HiveMetaStoreClient(hcatConf); System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " "); + System.setProperty(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, + MetastoreConf.getVar(hcatConf, MetastoreConf.ConfVars.WAREHOUSE)); + System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, + MetastoreConf.getVar(hcatConf, MetastoreConf.ConfVars.CONNECT_URL_KEY)); + System.setProperty(HiveConf.ConfVars.METASTOREURIS.varname, + MetastoreConf.getVar(hcatConf, MetastoreConf.ConfVars.THRIFT_URIS)); } @AfterClass diff --git itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java index accdc1f..f95e1c6 100644 --- itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java +++ itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java @@ -129,7 +129,7 @@ private void initialize(String cmRoot, String warehouseRoot, System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " "); - MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf); + MetaStoreTestUtils.startMetaStoreWithRetry(hiveConf, true); Path testPath = new Path(hiveWarehouseLocation); FileSystem testPathFileSystem = FileSystem.get(testPath.toUri(), hiveConf); diff --git itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index 23fcbe8..cbf73c6 100644 --- itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hive.llap.LlapItUtils; import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper; import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim; @@ -286,9 +287,6 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM hiveConf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, authType); } - String metaStoreURL = "jdbc:derby:;databaseName=" + baseDir.getAbsolutePath() + File.separator - + "test_metastore;create=true"; - if (isMetastoreSecure) { hiveConf.setVar(ConfVars.METASTORE_KERBEROS_PRINCIPAL, metastoreServerPrincipal); hiveConf.setVar(ConfVars.METASTORE_KERBEROS_KEYTAB_FILE, metastoreKeyTab); @@ -302,8 +300,6 @@ private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useM fs.mkdirs(wareHouseDir); setWareHouseDir(wareHouseDir.toString()); - System.setProperty(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, metaStoreURL); - hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, metaStoreURL); if (!usePortsFromConf) { // reassign a new port, just in case if one of the MR services grabbed the last one setBinaryPort(MetaStoreTestUtils.findFreePort()); @@ -342,6 +338,7 @@ public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType, boolean usePortsF public void start(Map confOverlay) throws Exception { if (isMetastoreRemote) { MetaStoreTestUtils.startMetaStoreWithRetry(getHiveConf()); + setWareHouseDir(MetastoreConf.getVar(getHiveConf(), MetastoreConf.ConfVars.WAREHOUSE)); } // Set confOverlay parameters diff --git standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index f007261..7f4b79a 100644 --- standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -1097,8 +1097,10 @@ public static void setHiveSiteLocation(URL location) { } public static Configuration newMetastoreConf() { + return newMetastoreConf(new Configuration()); + } - Configuration conf = new Configuration(); + public static Configuration newMetastoreConf(Configuration conf) { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); if (classLoader == null) { diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java index 1d12cf9..1f86db6 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java @@ -27,11 +27,15 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.events.EventCleanerTask; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; +import org.apache.hadoop.hive.metastore.utils.FileUtils; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,6 +44,7 @@ public class MetaStoreTestUtils { private static final Logger LOG = LoggerFactory.getLogger(MetaStoreTestUtils.class); + private static final String tmpDir = System.getProperty("test.tmp.dir"); public static final int RETRY_COUNT = 10; /** @@ -82,11 +87,23 @@ public static int startMetaStoreWithRetry(Configuration conf) throws Exception { return MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), conf); } + public static int startMetaStoreWithRetry(Configuration conf, boolean keepJdbcUri) + throws Exception { + return MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), conf, + keepJdbcUri); + } + public static int startMetaStoreWithRetry() throws Exception { return MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), MetastoreConf.newMetastoreConf()); } + public static int startMetaStoreWithRetry(HadoopThriftAuthBridge bridge, + Configuration conf) throws Exception { + return MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), + MetastoreConf.newMetastoreConf(), false); + } + /** * Starts a MetaStore instance with the given configuration and given bridge. * Tries to find a free port, and use it. If failed tries another port so the tests will not @@ -94,22 +111,46 @@ public static int startMetaStoreWithRetry() throws Exception { * instances will use different warehouse directories. * @param bridge The Thrift bridge to uses * @param conf The configuration to use + * @param keepJdbcUri If set to true, then the JDBC url is not changed * @return The port on which the MetaStore finally started * @throws Exception */ public static int startMetaStoreWithRetry(HadoopThriftAuthBridge bridge, - Configuration conf) throws Exception { + Configuration conf, boolean keepJdbcUri) throws Exception { Exception metaStoreException = null; String warehouseDir = MetastoreConf.getVar(conf, ConfVars.WAREHOUSE); + for (int tryCount = 0; tryCount < MetaStoreTestUtils.RETRY_COUNT; tryCount++) { try { int metaStorePort = findFreePort(); + // Setting metastore instance specific warehouse directory, postfixing with port Path postfixedWarehouseDir = new Path(warehouseDir, String.valueOf(metaStorePort)); MetastoreConf.setVar(conf, ConfVars.WAREHOUSE, postfixedWarehouseDir.toString()); + + String jdbcUrl = MetastoreConf.getVar(conf, ConfVars.CONNECT_URL_KEY); + if (!keepJdbcUri) { + // Setting metastore instance specific jdbc url postfixed with port + jdbcUrl = "jdbc:derby:;databaseName=" + tmpDir + File.separator + + "junit_metastore_db_" + metaStorePort + ";create=true"; + MetastoreConf.setVar(conf, ConfVars.CONNECT_URL_KEY, jdbcUrl); + } + + // Setting metastore instance specific metastore uri MetastoreConf.setVar(conf, ConfVars.THRIFT_URIS, "thrift://localhost:" + metaStorePort); MetaStoreTestUtils.startMetaStore(metaStorePort, bridge, conf); - LOG.error("MetaStore Thrift Server started on port: {} with warehouse dir: {}", - metaStorePort, postfixedWarehouseDir); + + // Creating warehouse dir, if not exists + Warehouse wh = new Warehouse(conf); + if (!wh.isDir(wh.getWhRoot())) { + FileSystem fs = wh.getWhRoot().getFileSystem(conf); + fs.mkdirs(wh.getWhRoot()); + fs.setPermission(wh.getWhRoot(), + new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); + LOG.info("MetaStore warehouse root dir ({}) is created", postfixedWarehouseDir); + } + + LOG.info("MetaStore Thrift Server started on port: {} with warehouse dir: {} with " + + "jdbcUrl: {}", metaStorePort, postfixedWarehouseDir, jdbcUrl); return metaStorePort; } catch (ConnectException ce) { metaStoreException = ce; diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java index 6f52a52..1912730 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java @@ -100,7 +100,7 @@ public abstract class TestHiveMetaStore { private static final Logger LOG = LoggerFactory.getLogger(TestHiveMetaStore.class); protected static HiveMetaStoreClient client; - protected static Configuration conf; + protected static Configuration conf = MetastoreConf.newMetastoreConf(); protected static Warehouse warehouse; protected static boolean isThriftClient = false; @@ -113,7 +113,6 @@ @Before public void setUp() throws Exception { - conf = MetastoreConf.newMetastoreConf(); warehouse = new Warehouse(conf); // set some values to use for getting conf. vars @@ -2928,7 +2927,7 @@ private void createFunction(String dbName, String funcName, String className, @Test public void testRetriableClientWithConnLifetime() throws Exception { - Configuration conf = MetastoreConf.newMetastoreConf(); + Configuration conf = MetastoreConf.newMetastoreConf(new Configuration(this.conf)); MetastoreConf.setTimeVar(conf, ConfVars.CLIENT_SOCKET_LIFETIME, 4, TimeUnit.SECONDS); MetaStoreTestUtils.setConfForStandloneMode(conf); long timeout = 5 * 1000; // Lets use a timeout more than the socket lifetime to simulate a reconnect diff --git standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java index e1f81bb..415988d 100644 --- standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java +++ standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java @@ -43,7 +43,6 @@ public void setUp() throws Exception { if (isServerStarted) { Assert.assertNotNull("Unable to connect to the MetaStore server", client); - MetastoreConf.setVar(conf, ConfVars.THRIFT_URIS, "thrift://localhost:" + port); return; }