diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 8bff2a9..f0bc2a7 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -58,16 +58,23 @@ protected Properties origProp; protected String auxJars; private static final Log l4j = LogFactory.getLog(HiveConf.class); + private static boolean loadMetastoreConfig = false; + private static boolean loadHiveServer2Config = false; private static URL hiveDefaultURL = null; private static URL hiveSiteURL = null; + private static URL hivemetastoreSiteUrl = null; + private static URL hiveServer2SiteUrl = null; + private static byte[] confVarByteArray = null; + private static final Map vars = new HashMap(); private final List restrictList = new ArrayList(); private boolean isWhiteListRestrictionEnabled = false; private final List modWhiteList = new ArrayList(); + static { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); if (classLoader == null) { @@ -78,6 +85,9 @@ // Look for hive-site.xml on the CLASSPATH and log its location if found. hiveSiteURL = classLoader.getResource("hive-site.xml"); + hivemetastoreSiteUrl = classLoader.getResource("hivemetastore-site.xml"); + hiveServer2SiteUrl = classLoader.getResource("hiveserver2-site.xml"); + for (ConfVars confVar : ConfVars.values()) { vars.put(confVar.varname, confVar); } @@ -1407,6 +1417,19 @@ private void initialize(Class cls) { addResource(hiveSiteURL); } + // load hivemetastore-site.xml if this is metastore and file exists + if (isLoadMetastoreConfig() && hivemetastoreSiteUrl != null) { + addResource(hivemetastoreSiteUrl); + } + + // load hiveserver2-site.xml if this is hiveserver2 and file exists + // metastore can be embedded within hiveserver2, in such cases + // the conf params in hiveserver2-site.xml will override whats defined + // in hivemetastore-site.xml + if (isLoadHiveServer2Config() && hiveServer2SiteUrl != null) { + addResource(hiveServer2SiteUrl); + } + // Overlay the values of any system properties whose names appear in the list of ConfVars applySystemProperties(); @@ -1554,6 +1577,15 @@ public static URL getHiveSiteLocation() { return hiveSiteURL; } + public static URL getMetastoreSiteLocation() { + return hivemetastoreSiteUrl; + } + + public static URL getHiveServer2SiteLocation() { + return hiveServer2SiteUrl; + } + + /** * @return the user name set in hadoop.job.ugi param or the current user from System * @throws IOException @@ -1722,4 +1754,20 @@ private void setupRestrictList() { restrictList.add(ConfVars.HIVE_IN_TEST.varname); restrictList.add(ConfVars.HIVE_CONF_RESTRICTED_LIST.varname); } + + public static boolean isLoadMetastoreConfig() { + return loadMetastoreConfig; + } + + public static void setLoadMetastoreConfig(boolean loadMetastoreConfig) { + HiveConf.loadMetastoreConfig = loadMetastoreConfig; + } + + public static boolean isLoadHiveServer2Config() { + return loadHiveServer2Config; + } + + public static void setLoadHiveServer2Config(boolean loadHiveServer2Config) { + HiveConf.loadHiveServer2Config = loadHiveServer2Config; + } } diff --git a/data/conf/hive-site.xml b/data/conf/hive-site.xml index 1c9c598..37ac8c0 100644 --- a/data/conf/hive-site.xml +++ b/data/conf/hive-site.xml @@ -221,4 +221,19 @@ false + + + hive.dummyparam.test.server.specific.config.override + from.hive-site.xml + Using dummy param to test server specific configuration + + + + hive.dummyparam.test.server.specific.config.hivesite + from.hive-site.xml + Using dummy param to test server specific configuration + + + + diff --git a/data/conf/hivemetastore-site.xml b/data/conf/hivemetastore-site.xml new file mode 100644 index 0000000..7ffbeaf --- /dev/null +++ b/data/conf/hivemetastore-site.xml @@ -0,0 +1,36 @@ + + + + + + + + + hive.dummyparam.test.server.specific.config.override + from.hivemetastore-site.xml + Using dummy param to test server specific configuration + + + + hive.dummyparam.test.server.specific.config.metastoresite + from.hivemetastore-site.xml + Using dummy param to test server specific configuration + + + + diff --git a/data/conf/hiveserver2-site.xml b/data/conf/hiveserver2-site.xml new file mode 100644 index 0000000..e86a39f --- /dev/null +++ b/data/conf/hiveserver2-site.xml @@ -0,0 +1,36 @@ + + + + + + + + + hive.dummyparam.test.server.specific.config.override + from.hiveserver2-site.xml + Using dummy param to test server specific configuration + + + + hive.dummyparam.test.server.specific.config.hiveserver2site + from.hiveserver2-site.xml + Using dummy param to test server specific configuration + + + + diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestServerSpecificConfig.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestServerSpecificConfig.java new file mode 100644 index 0000000..c19a906 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestServerSpecificConfig.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import java.io.IOException; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.server.HiveServer2; + +public class TestServerSpecificConfig extends TestCase { + + /** + * Verify if appropriate server configuration (metastore, hiveserver2) + * get loaded when the embedded clients are loaded + * + * @throws IOException + * @throws Throwable + */ + public void testHiveMetastoreSiteConfigs() throws IOException, Throwable { + HiveConf conf = new HiveConf(); + // check the properties expected in hive client without metastore + assertFalse(conf.isLoadMetastoreConfig()); + assertEquals("from.hive-site.xml", + conf.get("hive.dummyparam.test.server.specific.config.override")); + assertNull(conf.get("hive.dummyparam.test.server.specific.config.metastoresite")); + + + // check config properties expected with embedded metastore client + HiveMetaStoreClient msClient = new HiveMetaStoreClient(conf); + assertTrue(conf.isLoadMetastoreConfig()); + assertEquals("from.hivemetastore-site.xml", + conf.get("hive.dummyparam.test.server.specific.config.override")); + + assertEquals("from.hivemetastore-site.xml", + conf.get("hive.dummyparam.test.server.specific.config.metastoresite")); + + assertEquals("from.hive-site.xml", + conf.get("hive.dummyparam.test.server.specific.config.hivesite")); + + //verify that hiveserver2 config is not loaded + assertFalse(conf.isLoadHiveServer2Config()); + assertNull(conf.get("hive.dummyparam.test.server.specific.config.hiveserver2site")); + + // check if hiveserver2 config gets loaded when HS2 is started + + HiveServer2 hs2 = new HiveServer2(); + conf = new HiveConf(); + assertTrue(conf.isLoadHiveServer2Config()); + assertEquals("from.hiveserver2-site.xml", + conf.get("hive.dummyparam.test.server.specific.config.override")); + + assertEquals("from.hiveserver2-site.xml", + conf.get("hive.dummyparam.test.server.specific.config.hiveserver2site")); + + assertEquals("from.hivemetastore-site.xml", + conf.get("hive.dummyparam.test.server.specific.config.metastoresite")); + + assertEquals("from.hive-site.xml", + conf.get("hive.dummyparam.test.server.specific.config.hivesite")); + + + } +} \ No newline at end of file diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index acef599..b24ace8 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -5030,6 +5030,7 @@ public void parse(String[] args) { * @param args */ public static void main(String[] args) throws Throwable { + HiveConf.setLoadMetastoreConfig(true); HiveMetastoreCli cli = new HiveMetastoreCli(); cli.parse(args); final boolean isCliVerbose = cli.isVerbose(); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 664dccd..6b2378a 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -34,12 +34,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Random; -import java.util.Set; import javax.security.auth.login.LoginException; @@ -47,7 +45,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.common.ValidTxnListImpl; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.AbortTxnRequest; @@ -68,7 +65,6 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse; -import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse; import org.apache.hadoop.hive.metastore.api.GetPrincipalsInRoleRequest; import org.apache.hadoop.hive.metastore.api.GetPrincipalsInRoleResponse; import org.apache.hadoop.hive.metastore.api.GetRoleGrantsForPrincipalRequest; @@ -109,7 +105,6 @@ import org.apache.hadoop.hive.metastore.api.TableStatsRequest; import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; import org.apache.hadoop.hive.metastore.api.TxnAbortedException; -import org.apache.hadoop.hive.metastore.api.TxnInfo; import org.apache.hadoop.hive.metastore.api.TxnOpenException; import org.apache.hadoop.hive.metastore.api.Type; import org.apache.hadoop.hive.metastore.api.UnknownDBException; @@ -168,6 +163,14 @@ public HiveMetaStoreClient(HiveConf conf, HiveMetaHookLoader hookLoader) if (localMetaStore) { // instantiate the metastore server handler directly instead of connecting // through the network + + // Load the hivemetastore-site.xml if any first + if(!conf.isLoadMetastoreConfig()) { + // if the boolean was set, then the metastore conf would aleady be loaded + HiveConf.setLoadMetastoreConfig(true); + conf.addResource(conf.getMetastoreSiteLocation()); + } + client = HiveMetaStore.newHMSHandler("hive client", conf); isConnected = true; return; diff --git a/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java index 62b1d9c..9ee9785 100644 --- a/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java +++ b/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java @@ -32,6 +32,7 @@ public EmbeddedThriftBinaryCLIService() { super(new CLIService()); isEmbedded = true; + HiveConf.setLoadHiveServer2Config(true); cliService.init(new HiveConf()); cliService.start(); } diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java index e7ed267..0864dfb 100644 --- a/service/src/java/org/apache/hive/service/server/HiveServer2.java +++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager; -import org.apache.hadoop.hive.ql.exec.tez.TezSessionState; import org.apache.hive.common.util.HiveStringUtils; import org.apache.hive.service.CompositeService; import org.apache.hive.service.cli.CLIService; @@ -45,6 +44,7 @@ public HiveServer2() { super("HiveServer2"); + HiveConf.setLoadHiveServer2Config(true); } @@ -142,7 +142,7 @@ public static void main(String[] args) { // before any of the other core hive classes are loaded String initLog4jMessage = LogUtils.initHiveLog4j(); LOG.debug(initLog4jMessage); - + HiveStringUtils.startupShutdownMessage(HiveServer2.class, args, LOG); //log debug message from "oproc" after log4j initialize properly LOG.debug(oproc.getDebugMessage().toString());