diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 8bff2a9..6dcfa75 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -58,16 +58,23 @@ protected Properties origProp; protected String auxJars; private static final Log l4j = LogFactory.getLog(HiveConf.class); + private static boolean loadMetastoreConfig = false; + private static boolean loadHiveServer2Config = false; private static URL hiveDefaultURL = null; private static URL hiveSiteURL = null; + private static URL hivemetastoreSiteUrl = null; + private static URL hiveServer2SiteUrl = null; + private static byte[] confVarByteArray = null; + private static final Map vars = new HashMap(); private final List restrictList = new ArrayList(); private boolean isWhiteListRestrictionEnabled = false; private final List modWhiteList = new ArrayList(); + static { ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); if (classLoader == null) { @@ -78,6 +85,9 @@ // Look for hive-site.xml on the CLASSPATH and log its location if found. hiveSiteURL = classLoader.getResource("hive-site.xml"); + hivemetastoreSiteUrl = classLoader.getResource("hivemetastore-site.xml"); + hiveServer2SiteUrl = classLoader.getResource("hiveserver2-site.xml"); + for (ConfVars confVar : ConfVars.values()) { vars.put(confVar.varname, confVar); } @@ -1407,6 +1417,26 @@ private void initialize(Class cls) { addResource(hiveSiteURL); } + // if embedded metastore is to be used as per config so far + // then this is considered like the metastore server case + String msUri = this.getVar(HiveConf.ConfVars.METASTOREURIS); + if(HiveConfUtil.isEmbeddedMetaStore(msUri)){ + setLoadMetastoreConfig(true); + } + + // load hivemetastore-site.xml if this is metastore and file exists + if (isLoadMetastoreConfig() && hivemetastoreSiteUrl != null) { + addResource(hivemetastoreSiteUrl); + } + + // load hiveserver2-site.xml if this is hiveserver2 and file exists + // metastore can be embedded within hiveserver2, in such cases + // the conf params in hiveserver2-site.xml will override whats defined + // in hivemetastore-site.xml + if (isLoadHiveServer2Config() && hiveServer2SiteUrl != null) { + addResource(hiveServer2SiteUrl); + } + // Overlay the values of any system properties whose names appear in the list of ConfVars applySystemProperties(); @@ -1466,7 +1496,6 @@ private void initialize(Class cls) { setupRestrictList(); } - /** * Apply system properties to this object if the property name is defined in ConfVars * and the value is non-null and not an empty string. @@ -1554,6 +1583,15 @@ public static URL getHiveSiteLocation() { return hiveSiteURL; } + public static URL getMetastoreSiteLocation() { + return hivemetastoreSiteUrl; + } + + public static URL getHiveServer2SiteLocation() { + return hiveServer2SiteUrl; + } + + /** * @return the user name set in hadoop.job.ugi param or the current user from System * @throws IOException @@ -1722,4 +1760,20 @@ private void setupRestrictList() { restrictList.add(ConfVars.HIVE_IN_TEST.varname); restrictList.add(ConfVars.HIVE_CONF_RESTRICTED_LIST.varname); } + + public static boolean isLoadMetastoreConfig() { + return loadMetastoreConfig; + } + + public static void setLoadMetastoreConfig(boolean loadMetastoreConfig) { + HiveConf.loadMetastoreConfig = loadMetastoreConfig; + } + + public static boolean isLoadHiveServer2Config() { + return loadHiveServer2Config; + } + + public static void setLoadHiveServer2Config(boolean loadHiveServer2Config) { + HiveConf.loadHiveServer2Config = loadHiveServer2Config; + } } diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java new file mode 100644 index 0000000..0d3b94c --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConfUtil.java @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.conf; + +import org.apache.hadoop.hive.common.classification.InterfaceAudience.Private; + +/** + * Hive Configuration utils + */ +@Private +public class HiveConfUtil { + /** + * Check if metastore is being used in embedded mode. + * This utility function exists so that the logic for determining the mode is same + * in HiveConf and HiveMetaStoreClient + * @param msUri - metastore server uri + * @return + */ + public static boolean isEmbeddedMetaStore(String msUri) { + return (msUri == null) ? true : msUri.trim().isEmpty(); + } +} diff --git a/data/conf/hive-site.xml b/data/conf/hive-site.xml index 1c9c598..37ac8c0 100644 --- a/data/conf/hive-site.xml +++ b/data/conf/hive-site.xml @@ -221,4 +221,19 @@ false + + + hive.dummyparam.test.server.specific.config.override + from.hive-site.xml + Using dummy param to test server specific configuration + + + + hive.dummyparam.test.server.specific.config.hivesite + from.hive-site.xml + Using dummy param to test server specific configuration + + + + diff --git a/data/conf/hivemetastore-site.xml b/data/conf/hivemetastore-site.xml new file mode 100644 index 0000000..2e92678 --- /dev/null +++ b/data/conf/hivemetastore-site.xml @@ -0,0 +1,42 @@ + + + + + + + + + hive.dummyparam.test.server.specific.config.override + from.hivemetastore-site.xml + Using dummy param to test server specific configuration + + + + hive.dummyparam.test.server.specific.config.metastoresite + from.hivemetastore-site.xml + Using dummy param to test server specific configuration + + + + hive.conf.restricted.list + from.hivemetastore-site.xml + Using property defined in HiveConf.ConfVars to test System property overriding + + + + diff --git a/data/conf/hiveserver2-site.xml b/data/conf/hiveserver2-site.xml new file mode 100644 index 0000000..ff5a057 --- /dev/null +++ b/data/conf/hiveserver2-site.xml @@ -0,0 +1,42 @@ + + + + + + + + + hive.dummyparam.test.server.specific.config.override + from.hiveserver2-site.xml + Using dummy param to test server specific configuration + + + + hive.dummyparam.test.server.specific.config.hiveserver2site + from.hiveserver2-site.xml + Using dummy param to test server specific configuration + + + + hive.conf.restricted.list + from.hiveserver2-site.xml + Using property defined in HiveConf.ConfVars to test System property overriding + + + + diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestServerSpecificConfig.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestServerSpecificConfig.java new file mode 100644 index 0000000..6ca673a --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestServerSpecificConfig.java @@ -0,0 +1,195 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.metastore; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.net.URL; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.service.server.HiveServer2; +import org.junit.Before; +import org.junit.Test; + +public class TestServerSpecificConfig { + + private static URL oldDefaultHiveSite = HiveConf.getHiveSiteLocation(); + + /** + * Verify if appropriate server configuration (metastore, hiveserver2) get + * loaded when the embedded clients are loaded + * + * Checks values used in the configs used for testing. + * + * @throws IOException + * @throws Throwable + */ + @Test + public void testServerConfigsEmbeddedMetastore() throws IOException, Throwable { + + // set hive-site.xml to default hive-site.xml that has embedded metastore + HiveConf.setHiveSiteLocation(oldDefaultHiveSite); + + HiveConf conf = new HiveConf(); + + // check config properties expected with embedded metastore client + assertTrue(HiveConf.isLoadMetastoreConfig()); + assertEquals("from.hivemetastore-site.xml", + conf.get("hive.dummyparam.test.server.specific.config.override")); + + assertEquals("from.hivemetastore-site.xml", + conf.get("hive.dummyparam.test.server.specific.config.metastoresite")); + + assertEquals("from.hive-site.xml", + conf.get("hive.dummyparam.test.server.specific.config.hivesite")); + + // verify that hiveserver2 config is not loaded + assertFalse(HiveConf.isLoadHiveServer2Config()); + assertNull(conf.get("hive.dummyparam.test.server.specific.config.hiveserver2site")); + + // check if hiveserver2 config gets loaded when HS2 is started + new HiveServer2(); + conf = new HiveConf(); + verifyHS2ConfParams(conf); + + assertEquals("from.hivemetastore-site.xml", + conf.get("hive.dummyparam.test.server.specific.config.metastoresite")); + } + + private void verifyHS2ConfParams(HiveConf conf) { + assertTrue(HiveConf.isLoadHiveServer2Config()); + assertEquals("from.hiveserver2-site.xml", + conf.get("hive.dummyparam.test.server.specific.config.override")); + + assertEquals("from.hiveserver2-site.xml", + conf.get("hive.dummyparam.test.server.specific.config.hiveserver2site")); + + assertEquals("from.hive-site.xml", + conf.get("hive.dummyparam.test.server.specific.config.hivesite")); + } + + /** + * Ensure that system properties still get precedence. Config params set as + * -hiveconf on commandline get set as system properties They should have the + * final say + */ + @Test + public void testSystemPropertyPrecedence() { + // Using property defined in HiveConf.ConfVars to test System property + // overriding + final String OVERRIDE_KEY = "hive.conf.restricted.list"; + try { + HiveConf.setHiveSiteLocation(oldDefaultHiveSite); + System.setProperty(OVERRIDE_KEY, "from.sysprop"); + HiveConf conf = new HiveConf(); + // ensure metatore site.xml does not get to override this + assertEquals("from.sysprop", conf.get(OVERRIDE_KEY)); + + // get HS2 site.xml loaded + new HiveServer2(); + conf = new HiveConf(); + assertTrue(HiveConf.isLoadHiveServer2Config()); + // ensure hiveserver2 site.xml does not get to override this + assertEquals("from.sysprop", conf.get(OVERRIDE_KEY)); + + } finally { + System.getProperties().remove(OVERRIDE_KEY); + } + } + + @Before + public void resetDefaults() throws SecurityException, IllegalArgumentException, + NoSuchFieldException, IllegalAccessException { + // re-set the static variables in HiveConf to default values + + // set load server conf booleans to false + HiveConf.setLoadMetastoreConfig(false); + HiveConf.setLoadHiveServer2Config(false); + + } + + /** + * Test to ensure that HiveConf does not try to load hivemetastore-site.xml, + * when remote metastore is used. + * + * @throws IOException + * @throws Throwable + */ + @Test + public void testHiveMetastoreRemoteConfig() throws IOException, Throwable { + // switch to hive-site.xml with remote metastore + setHiveSiteWithRemoteMetastore(); + + // Set HiveConf statics to default values + resetDefaults(); + + // create hiveconf again to run initialization code, to see if value changes + HiveConf conf = new HiveConf(); + + // check the properties expected in hive client without metastore + verifyMetastoreConfNotLoaded(conf); + assertEquals("from.hive-site.xml", + conf.get("hive.dummyparam.test.server.specific.config.override")); + + // get HS2 site.xml loaded + new HiveServer2(); + conf = new HiveConf(); + verifyHS2ConfParams(conf); + verifyMetastoreConfNotLoaded(conf); + } + + private void verifyMetastoreConfNotLoaded(HiveConf conf) { + assertFalse(HiveConf.isLoadMetastoreConfig()); + assertNull(conf.get("hive.dummyparam.test.server.specific.config.metastoresite")); + } + + /** + * Set new hive-site.xml file location that has remote metastore config + * + * @throws IOException + */ + private void setHiveSiteWithRemoteMetastore() throws IOException { + // new *hive-site.xml file + String newConfFile = System.getProperty("test.tmp.dir") + File.separator + + this.getClass().getSimpleName() + "hive-site.xml"; + + // create a new conf file, using contents from current one + // modifying the meastore.uri property + File hiveSite = new File(newConfFile); + FileOutputStream out = new FileOutputStream(hiveSite); + HiveConf.setHiveSiteLocation(oldDefaultHiveSite); + HiveConf defaultHiveConf = new HiveConf(); + defaultHiveConf.setVar(ConfVars.METASTOREURIS, "dummyvalue"); + // reset to the hive-site.xml values for following param + defaultHiveConf.set("hive.dummyparam.test.server.specific.config.override", + "from.hive-site.xml"); + defaultHiveConf.unset("hive.dummyparam.test.server.specific.config.metastoresite"); + defaultHiveConf.writeXml(out); + + // set the new hive-site.xml + HiveConf.setHiveSiteLocation(hiveSite.toURI().toURL()); + } + +} \ No newline at end of file diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index acef599..b24ace8 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -5030,6 +5030,7 @@ public void parse(String[] args) { * @param args */ public static void main(String[] args) throws Throwable { + HiveConf.setLoadMetastoreConfig(true); HiveMetastoreCli cli = new HiveMetastoreCli(); cli.parse(args); final boolean isCliVerbose = cli.isVerbose(); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 664dccd..df034cf 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -34,12 +34,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Random; -import java.util.Set; import javax.security.auth.login.LoginException; @@ -47,9 +45,9 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.common.ValidTxnListImpl; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveConfUtil; import org.apache.hadoop.hive.metastore.api.AbortTxnRequest; import org.apache.hadoop.hive.metastore.api.AddPartitionsRequest; import org.apache.hadoop.hive.metastore.api.AddPartitionsResult; @@ -68,7 +66,6 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Function; import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse; -import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse; import org.apache.hadoop.hive.metastore.api.GetPrincipalsInRoleRequest; import org.apache.hadoop.hive.metastore.api.GetPrincipalsInRoleResponse; import org.apache.hadoop.hive.metastore.api.GetRoleGrantsForPrincipalRequest; @@ -109,7 +106,6 @@ import org.apache.hadoop.hive.metastore.api.TableStatsRequest; import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; import org.apache.hadoop.hive.metastore.api.TxnAbortedException; -import org.apache.hadoop.hive.metastore.api.TxnInfo; import org.apache.hadoop.hive.metastore.api.TxnOpenException; import org.apache.hadoop.hive.metastore.api.Type; import org.apache.hadoop.hive.metastore.api.UnknownDBException; @@ -164,7 +160,7 @@ public HiveMetaStoreClient(HiveConf conf, HiveMetaHookLoader hookLoader) this.conf = conf; String msUri = conf.getVar(HiveConf.ConfVars.METASTOREURIS); - localMetaStore = (msUri == null) ? true : msUri.trim().isEmpty(); + localMetaStore = HiveConfUtil.isEmbeddedMetaStore(msUri); if (localMetaStore) { // instantiate the metastore server handler directly instead of connecting // through the network diff --git a/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java index 62b1d9c..9ee9785 100644 --- a/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java +++ b/service/src/java/org/apache/hive/service/cli/thrift/EmbeddedThriftBinaryCLIService.java @@ -32,6 +32,7 @@ public EmbeddedThriftBinaryCLIService() { super(new CLIService()); isEmbedded = true; + HiveConf.setLoadHiveServer2Config(true); cliService.init(new HiveConf()); cliService.start(); } diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java index e7ed267..0864dfb 100644 --- a/service/src/java/org/apache/hive/service/server/HiveServer2.java +++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -25,7 +25,6 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager; -import org.apache.hadoop.hive.ql.exec.tez.TezSessionState; import org.apache.hive.common.util.HiveStringUtils; import org.apache.hive.service.CompositeService; import org.apache.hive.service.cli.CLIService; @@ -45,6 +44,7 @@ public HiveServer2() { super("HiveServer2"); + HiveConf.setLoadHiveServer2Config(true); } @@ -142,7 +142,7 @@ public static void main(String[] args) { // before any of the other core hive classes are loaded String initLog4jMessage = LogUtils.initHiveLog4j(); LOG.debug(initLog4jMessage); - + HiveStringUtils.startupShutdownMessage(HiveServer2.class, args, LOG); //log debug message from "oproc" after log4j initialize properly LOG.debug(oproc.getDebugMessage().toString());