From 9de3364df9ad65467b82821d280f2f183c058f33 Mon Sep 17 00:00:00 2001 From: ycq Date: Wed, 7 Mar 2018 17:10:38 +0800 Subject: [PATCH] KYLIN-3277 Kylin should override hiveconf settings when connecting to hive using jdbc --- .../apache/kylin/common/util/HiveCmdBuilder.java | 47 +--------- .../kylin/common/util/HiveConfigurationUtil.java | 101 +++++++++++++++++++++ .../kylin/source/hive/BeelineHiveClient.java | 16 +++- 3 files changed, 115 insertions(+), 49 deletions(-) create mode 100644 core-common/src/main/java/org/apache/kylin/common/util/HiveConfigurationUtil.java diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java index c33a7fb..cb00a18 100644 --- a/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java +++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveCmdBuilder.java @@ -46,12 +46,12 @@ public class HiveCmdBuilder { } private KylinConfig kylinConfig; - final private Map hiveConfProps = new HashMap<>(); + final private Map hiveConfProps; final private ArrayList statements = Lists.newArrayList(); public HiveCmdBuilder() { kylinConfig = KylinConfig.getInstanceFromEnv(); - loadHiveConfiguration(); + hiveConfProps = HiveConfigurationUtil.loadHiveConfiguration(); } public String build() { @@ -169,47 +169,4 @@ public class HiveCmdBuilder { return build(); } - private void loadHiveConfiguration() { - - File hiveConfFile; - String hiveConfFileName = (HIVE_CONF_FILENAME + ".xml"); - String path = System.getProperty(KylinConfig.KYLIN_CONF); - - if (StringUtils.isNotEmpty(path)) { - hiveConfFile = new File(path, hiveConfFileName); - } else { - path = KylinConfig.getKylinHome(); - if (StringUtils.isEmpty(path)) { - logger.error("KYLIN_HOME is not set, can not locate hive conf: {}.xml", HIVE_CONF_FILENAME); - return; - } - hiveConfFile = new File(path + File.separator + "conf", hiveConfFileName); - } - - if (!hiveConfFile.exists()) { - throw new RuntimeException("Missing config file: " + hiveConfFile.getAbsolutePath()); - } - - String fileUrl = OptionsHelper.convertToFileURL(hiveConfFile.getAbsolutePath()); - - try { - File file = new File(fileUrl); - if (file.exists()) { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - Document doc = builder.parse(file); - NodeList nl = doc.getElementsByTagName("property"); - hiveConfProps.clear(); - for (int i = 0; i < nl.getLength(); i++) { - String key = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue(); - String value = doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue(); - if (!key.equals("tmpjars")) { - hiveConfProps.put(key, value); - } - } - } - } catch (Exception e) { - throw new RuntimeException("Failed to parse hive conf file ", e); - } - } } \ No newline at end of file diff --git a/core-common/src/main/java/org/apache/kylin/common/util/HiveConfigurationUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/HiveConfigurationUtil.java new file mode 100644 index 0000000..a8cba61 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/util/HiveConfigurationUtil.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.common.util; + +import org.apache.commons.lang.StringUtils; +import org.apache.kylin.common.KylinConfig; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.NodeList; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import java.io.File; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.apache.kylin.common.util.HiveCmdBuilder.HIVE_CONF_FILENAME; + +/** + * @author ycq + * @since 2018-03-05 + */ +public class HiveConfigurationUtil { + + private static final org.slf4j.Logger logger = LoggerFactory.getLogger(HiveConfigurationUtil.class); + private static final String HIVE_CONF_PREFIX = "hiveconf:"; + + public static Properties loadHiveJDBCProperties() { + Map hiveConfiguration = loadHiveConfiguration(); + Properties ret = new Properties(); + for (Map.Entry entry : hiveConfiguration.entrySet()) { + ret.put(HIVE_CONF_PREFIX + entry.getKey(), entry.getValue()); + } + return ret; + } + + public static Map loadHiveConfiguration() { + Map hiveConfProps = new HashMap<>(); + File hiveConfFile; + String hiveConfFileName = (HIVE_CONF_FILENAME + ".xml"); + String path = System.getProperty(KylinConfig.KYLIN_CONF); + + if (StringUtils.isNotEmpty(path)) { + hiveConfFile = new File(path, hiveConfFileName); + } else { + path = KylinConfig.getKylinHome(); + if (StringUtils.isEmpty(path)) { + logger.error("KYLIN_HOME is not set, can not locate hive conf: {}.xml", HIVE_CONF_FILENAME); + return hiveConfProps; + } + hiveConfFile = new File(path + File.separator + "conf", hiveConfFileName); + } + + if (!hiveConfFile.exists()) { + throw new RuntimeException("Failed to read " + HIVE_CONF_FILENAME + ".xml"); + } + + String fileUrl = OptionsHelper.convertToFileURL(hiveConfFile.getAbsolutePath()); + + try { + File file = new File(fileUrl); + if (file.exists()) { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document doc = builder.parse(file); + NodeList nl = doc.getElementsByTagName("property"); + hiveConfProps.clear(); + for (int i = 0; i < nl.getLength(); i++) { + String key = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue(); + String value = doc.getElementsByTagName("value").item(i).getFirstChild().getNodeValue(); + if (!key.equals("tmpjars")) { + hiveConfProps.put(key, value); + } + } + } + } catch (Exception e) { + throw new RuntimeException("Failed to parse hive conf file ", e); + } + return hiveConfProps; + } + +} + + diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java index 747b1bb..593ad96 100644 --- a/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java +++ b/source-hive/src/main/java/org/apache/kylin/source/hive/BeelineHiveClient.java @@ -26,25 +26,30 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.List; +import java.util.Properties; import org.apache.commons.lang3.StringUtils; import org.apache.kylin.common.util.DBUtils; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import org.apache.kylin.common.util.HiveConfigurationUtil; public class BeelineHiveClient implements IHiveClient { + private static final String HIVE_AUTH_USER = "user"; + private static final String HIVE_AUTH_PASSWD = "password"; private Connection cnct; private Statement stmt; private DatabaseMetaData metaData; + public BeelineHiveClient(String beelineParams) { if (StringUtils.isEmpty(beelineParams)) { throw new IllegalArgumentException("BeelineParames cannot be empty"); } String[] splits = StringUtils.split(beelineParams); - String url = null, username = null, password = null; + String url = "", username = "", password = ""; for (int i = 0; i < splits.length; i++) { if ("-u".equals(splits[i])) { url = stripQuotes(splits[i + 1]); @@ -56,13 +61,16 @@ public class BeelineHiveClient implements IHiveClient { password = stripQuotes(splits[i + 1]); } } - this.init(url, username, password); + Properties jdbcProperties = HiveConfigurationUtil.loadHiveJDBCProperties(); + jdbcProperties.put(HIVE_AUTH_PASSWD, password); + jdbcProperties.put(HIVE_AUTH_USER, username); + this.init(url, jdbcProperties); } - private void init(String url, String username, String password) { + private void init(String url, Properties hiveProperties) { try { Class.forName("org.apache.hive.jdbc.HiveDriver"); - cnct = DriverManager.getConnection(url, username, password); + cnct = DriverManager.getConnection(url, hiveProperties); stmt = cnct.createStatement(); metaData = cnct.getMetaData(); } catch (SQLException | ClassNotFoundException e) { -- 2.10.1 (Apple Git-78)