commit e29ba4aaf6a4c6e3e21ff7822d78ad11a0bd6560 Author: Vihang Karajgaonkar Date: Wed Jun 22 12:48:45 2016 -0700 HIVE-14063 : Add beeline connection configuration file to automatically connect to HS2 using beeline This change adds a new optional configuration file for Beeline. If this file is present at predefined locations, Beeline will attempt to create the connection url using the hive-site.xml found in classpath and another user-specific configuration file. Beeline then connects automatically using the url generated based on these configuration files. The main objective of the change is to provide user another way to connect to the HiveServer2 without providing the connection url everytime. The configuration file uses hadoop xml format so that we can support encryption/obfuscation using hadoop credential manager API in the future. Properties in the user-specific configuration file override the properties derived from hive-site.xml. Tested using newly added unit tests and itests in various Hiveserver2 configurations. diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index 8e65e3987398531cce5c65c383762cf49a52c578..f0d4f4f078a93e6bb8032efc5d863795b7f5e92e 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -28,6 +28,7 @@ import java.io.EOFException; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; @@ -92,6 +93,11 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.io.IOUtils; import org.apache.hive.beeline.cli.CliOptionsProcessor; +import org.apache.hive.beeline.hs2connection.BeelineHS2ConnectionFileParseException; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileUtils; +import org.apache.hive.beeline.hs2connection.UserHS2ConnectionFileParser; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileParser; +import org.apache.hive.beeline.hs2connection.HiveSiteHS2ConnectionFileParser; import org.apache.thrift.transport.TTransportException; import org.apache.hive.jdbc.Utils; @@ -278,7 +284,6 @@ "org.apache.hadoop.hive.jdbc.HiveDriver", })); - static { try { Class.forName("jline.console.ConsoleReader"); @@ -768,20 +773,10 @@ int initArgs(String[] args) { } if (!commands.isEmpty() && getOpts().getScriptFile() != null) { - System.err.println("The '-e' and '-f' options cannot be specified simultaneously"); + error("The '-e' and '-f' options cannot be specified simultaneously"); return 1; } - // TODO: temporary disable this for easier debugging - /* - if (url == null) { - url = BEELINE_DEFAULT_JDBC_URL; - } - if (driver == null) { - driver = BEELINE_DEFAULT_JDBC_DRIVER; - } - */ - if (url != null) { if (user == null) { user = Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_USER); @@ -800,17 +795,16 @@ int initArgs(String[] args) { // load property file String propertyFile = cl.getOptionValue("property-file"); if (propertyFile != null) { - try { - this.consoleReader = new ConsoleReader(); - } catch (IOException e) { - handleException(e); - } if (!dispatch("!properties " + propertyFile)) { exit = true; return 1; } + } else { + //checks if default hs2 connection configuration file is present + //and uses it to connect if found + //no-op if the file is not present + defaultBeelineConnect(); } - int code = 0; if (!commands.isEmpty()) { for (Iterator i = commands.iterator(); i.hasNext();) { @@ -825,7 +819,6 @@ int initArgs(String[] args) { return code; } - private void setHiveConfVar(String key, String val) { getOpts().getHiveConfVariables().put(key, val); if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key) && "mr".equals(val)) { @@ -882,6 +875,7 @@ public int begin(String[] args, InputStream inputStream) throws IOException { try { ConsoleReader reader = getConsoleReader(inputStream); + setConsoleReader(reader); if (isBeeLine) { int code = initArgs(args); if (code != 0) { @@ -906,13 +900,65 @@ public int begin(String[] args, InputStream inputStream) throws IOException { } catch (Exception e) { // ignore } - return execute(reader, false); } finally { close(); } } + /* + * Attempts to make a connection using default HS2 connection config file if available + * + */ + public int defaultBeelineConnect() { + String url; + try { + url = getDefaultConnectionUrl(); + if (url == null) { + debug("Default hs2 connection config file not found"); + return ERRNO_OK; + } + } catch (BeelineHS2ConnectionFileParseException e) { + error(e); + return ERRNO_OTHER; + } + if (!dispatch("!connect " + url)) { + return ERRNO_OTHER; + } + return ERRNO_OK; + } + + + public String getDefaultConnectionUrl() throws BeelineHS2ConnectionFileParseException { + HS2ConnectionFileParser defaultParser = getDefaultHS2ConnectionFileParser(); + if(defaultParser.getFileLocation() == null) { + //nothing to do if there is no user HS2 connection configuration file + return null; + } + //get the connection properties from user specific config file + Properties userConnectionProperties = defaultParser.getConnectionProperties(); + //load the HS2 connection url properties from hive-site.xml if it is present in the classpath + HS2ConnectionFileParser hiveSiteParser = getHiveSiteHS2ConnectionFileParser(); + Properties hiveSiteConnectionProperties = hiveSiteParser.getConnectionProperties(); + //add/override properties found from hive-site with user-specific properties + for (String key : userConnectionProperties.stringPropertyNames()) { + if (hiveSiteConnectionProperties.containsKey(key)) { + debug("Overriding connection url property " + key + " from user connection configuration file"); + } + hiveSiteConnectionProperties.setProperty(key, userConnectionProperties.getProperty(key)); + } + //return the url based on the aggregated connection properties + return HS2ConnectionFileUtils.getUrl(hiveSiteConnectionProperties); + } + + public HS2ConnectionFileParser getDefaultHS2ConnectionFileParser() { + return new UserHS2ConnectionFileParser(); + } + + public HS2ConnectionFileParser getHiveSiteHS2ConnectionFileParser() { + return new HiveSiteHS2ConnectionFileParser(); + } + int runInit() { String initFiles[] = getOpts().getInitFiles(); if (initFiles != null && initFiles.length != 0) { diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java b/beeline/src/java/org/apache/hive/beeline/Commands.java index 2f3ec134098dfa3767bab9545438d1f38f11697c..1cffc9081e8ab1ae5a88c10ca43bc3ead0f66794 100644 --- a/beeline/src/java/org/apache/hive/beeline/Commands.java +++ b/beeline/src/java/org/apache/hive/beeline/Commands.java @@ -22,12 +22,6 @@ */ package org.apache.hive.beeline; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveVariableSource; -import org.apache.hadoop.hive.conf.SystemVariables; -import org.apache.hadoop.hive.conf.VariableSubstitution; -import org.apache.hadoop.io.IOUtils; - import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -46,8 +40,8 @@ import java.sql.Driver; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.sql.SQLWarning; +import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -60,6 +54,12 @@ import java.util.TreeSet; import org.apache.hadoop.hive.common.cli.ShellCmdExecutor; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveVariableSource; +import org.apache.hadoop.hive.conf.SystemVariables; +import org.apache.hadoop.hive.conf.VariableSubstitution; +import org.apache.hadoop.io.IOUtils; import org.apache.hive.jdbc.HiveStatement; import org.apache.hive.jdbc.Utils; import org.apache.hive.jdbc.Utils.JdbcConnectionParams; @@ -1465,7 +1465,6 @@ private String getProperty(Properties props, String[] keys) { return null; } - public boolean connect(Properties props) throws IOException { String url = getProperty(props, new String[] { JdbcConnectionParams.PROPERTY_URL, @@ -1507,13 +1506,14 @@ public boolean connect(Properties props) throws IOException { beeLine.info("Connecting to " + url); if (Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_PRINCIPAL) == null) { + String urlForPrompt = url.substring(0, url.contains(";") ? url.indexOf(';') : url.length()); if (username == null) { - username = beeLine.getConsoleReader().readLine("Enter username for " + url + ": "); + username = beeLine.getConsoleReader().readLine("Enter username for " + urlForPrompt + ": "); } props.setProperty(JdbcConnectionParams.AUTH_USER, username); if (password == null || password.length() == 0) { - password = beeLine.getConsoleReader().readLine("Enter password for " + url + ": ", - new Character('*')); + password = beeLine.getConsoleReader().readLine("Enter password for " + urlForPrompt + ": ", + new Character('*')); } props.setProperty(JdbcConnectionParams.AUTH_PASSWD, password); } @@ -1539,7 +1539,6 @@ public boolean connect(Properties props) throws IOException { } } - public boolean rehash(String line) { try { if (!(beeLine.assertConnection())) { diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/BeelineHS2ConnectionFileParseException.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/BeelineHS2ConnectionFileParseException.java new file mode 100644 index 0000000000000000000000000000000000000000..6e6018631a11782130c1375fa1cd398f216f9c17 --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/BeelineHS2ConnectionFileParseException.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +public class BeelineHS2ConnectionFileParseException extends Exception { + private static final long serialVersionUID = -748635913718300617L; + + BeelineHS2ConnectionFileParseException(String msg, Exception e) { + super(msg, e); + } + + public BeelineHS2ConnectionFileParseException(String msg) { + super(msg); + } +} diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileParser.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileParser.java new file mode 100644 index 0000000000000000000000000000000000000000..af38138eb8e5a008a7e368bb2a432ac0209aff04 --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileParser.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.util.Properties; + +/** + * HS2ConnectionFileParser provides a interface to be used by Beeline to parse a configuration (file + * or otherwise) to return a Java Properties object which contain key value pairs to be used to construct + * connection URL automatically for the Beeline connection + */ +public interface HS2ConnectionFileParser { + // prefix string used for the keys + public static final String BEELINE_CONNECTION_PROPERTY_PREFIX = "beeline.hs2.connection."; + + /** + * Returns a Java properties object which contain the key value pairs which can be used in the + * Beeline connection URL

+ * The properties returned must include url_prefix and hosts

+ * Following are some examples of the URLs and returned properties object

+ *

+ * + * @return Properties object which contain connection URL properties for Beeline connection + * @throws BeelineHS2ConnectionFileParseException if there is invalid key with appropriate message + */ + Properties getConnectionProperties() throws BeelineHS2ConnectionFileParseException; + + /** + * + * @return returns the location of file used to generate the connection properties + */ + String getFileLocation(); +} \ No newline at end of file diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileUtils.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..a377f4244c01249127b7580890ecd8d80f5e21b9 --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileUtils.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +public class HS2ConnectionFileUtils { + + public static String getUrl(Properties props) throws BeelineHS2ConnectionFileParseException { + if (props == null || props.isEmpty()) { + return null; + } + // use remove instead of get so that it is not parsed again + // in the for loop below + String urlPrefix = (String) props.remove("url_prefix"); + if (urlPrefix == null || urlPrefix.isEmpty()) { + throw new BeelineHS2ConnectionFileParseException("url_prefix parameter cannot be empty"); + } + + String hosts = (String) props.remove("hosts"); + if (hosts == null || hosts.isEmpty()) { + throw new BeelineHS2ConnectionFileParseException("hosts parameter cannot be empty"); + } + String defaultDB = (String) props.remove("defaultDB"); + if (defaultDB == null) { + defaultDB = "default"; + } + // collect the hiveConfList and HiveVarList separately so that they can be + // appended once all the session list are added to the url + String hiveConfProperties = ""; + if (props.containsKey("hiveconf")) { + hiveConfProperties = extractHiveVariables((String) props.remove("hiveconf"), true); + } + + String hiveVarProperties = ""; + if (props.containsKey("hivevar")) { + hiveVarProperties = extractHiveVariables((String) props.remove("hivevar"), false); + } + StringBuilder urlSb = new StringBuilder(); + urlSb.append(urlPrefix.trim()); + urlSb.append(hosts.trim()); + urlSb.append(File.separator); + urlSb.append(defaultDB.trim()); + List keys = new ArrayList<>(props.stringPropertyNames()); + // sorting the keys from the properties helps to create + // a deterministic url which is tested for various configuration in + // TestHS2ConnectionConfigFileManager + Collections.sort(keys); + for (String propertyName : keys) { + urlSb.append(";"); + urlSb.append(propertyName); + urlSb.append("="); + urlSb.append(props.getProperty(propertyName)); + } + if (!hiveConfProperties.isEmpty()) { + urlSb.append(hiveConfProperties.toString()); + } + if (!hiveVarProperties.isEmpty()) { + urlSb.append(hiveVarProperties.toString()); + } + return urlSb.toString(); + } + + private static String extractHiveVariables(String propertyValue, boolean isHiveConf) + throws BeelineHS2ConnectionFileParseException { + StringBuilder hivePropertiesList = new StringBuilder(); + String delimiter; + if (isHiveConf) { + delimiter = "?"; + } else { + delimiter = "#"; + } + hivePropertiesList.append(delimiter); + addPropertyValues(propertyValue, hivePropertiesList); + return hivePropertiesList.toString(); + } + + private static void addPropertyValues(String value, StringBuilder hivePropertiesList) + throws BeelineHS2ConnectionFileParseException { + if (!value.contains("=")) { + throw new BeelineHS2ConnectionFileParseException( + "Unable to parse " + value + " in hs2 connection config file"); + } + // There could be multiple keyValuePairs separated by comma + String[] values = value.split(","); + boolean first = true; + for (String keyValuePair : values) { + String[] keyValue = keyValuePair.split("="); + if (keyValue.length != 2) { + throw new BeelineHS2ConnectionFileParseException("Unable to parse " + keyValuePair + + " in hs2 connection config file"); + } + if (!first) { + hivePropertiesList.append(";"); + } + first = false; + hivePropertiesList.append(keyValue[0].trim()); + hivePropertiesList.append("="); + hivePropertiesList.append(keyValue[1].trim()); + } + } +} diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/HiveSiteHS2ConnectionFileParser.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/HiveSiteHS2ConnectionFileParser.java new file mode 100644 index 0000000000000000000000000000000000000000..0992be13eeb9e2cd3060adaffd4f361335fddba0 --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/HiveSiteHS2ConnectionFileParser.java @@ -0,0 +1,169 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.net.InetAddress; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.ServerUtils; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; + +/* + * Looks for a hive-site.xml from the classpath. If found this class parses the hive-site.xml + * to return a set of connection properties which can be used to construct the connection url + * for Beeline connection + */ +public class HiveSiteHS2ConnectionFileParser implements HS2ConnectionFileParser { + private Configuration conf; + private final URL hiveSiteURI; + private final String TRUSTSTORE_PASS_PROP = "javax.net.ssl.trustStorePassword"; + private final String TRUSTSTORE_PROP = "javax.net.ssl.trustStore"; + private static final Logger log = LoggerFactory.getLogger(HiveSiteHS2ConnectionFileParser.class); + + public HiveSiteHS2ConnectionFileParser() { + hiveSiteURI = HiveConf.getHiveSiteLocation(); + conf = new Configuration(); + if(hiveSiteURI == null) { + log.debug("hive-site.xml not found for constructing the connection URL"); + } else { + log.info("Using hive-site.xml at " + hiveSiteURI); + conf.addResource(hiveSiteURI); + } + } + + @VisibleForTesting + public void setHiveConf(HiveConf hiveConf) { + this.conf = hiveConf; + } + + @Override + public Properties getConnectionProperties() throws BeelineHS2ConnectionFileParseException { + Properties props = new Properties(); + props.setProperty("url_prefix", "jdbc:hive2://"); + addHosts(props); + addSSL(props); + addKerberos(props); + addHttp(props); + return props; + } + + private void addSSL(Properties props) { + if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_SERVER2_USE_SSL)) { + return; + } else { + props.setProperty("ssl", "true"); + } + String truststore = System.getenv(TRUSTSTORE_PROP); + if (truststore != null && truststore.isEmpty()) { + props.setProperty("sslTruststore", truststore); + } + String trustStorePassword = System.getenv(TRUSTSTORE_PASS_PROP); + if (trustStorePassword != null && !trustStorePassword.isEmpty()) { + props.setProperty("trustStorePassword", trustStorePassword); + } + String saslQop = HiveConf.getVar(conf, ConfVars.HIVE_SERVER2_THRIFT_SASL_QOP); + if (!"auth".equalsIgnoreCase(saslQop)) { + props.setProperty("sasl.qop", saslQop); + } + } + + private void addKerberos(Properties props) { + if ("KERBEROS".equals(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION))) { + props.setProperty("principal", + HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL)); + } + } + + private void addHttp(Properties props) { + if ("http".equalsIgnoreCase(HiveConf.getVar(conf, ConfVars.HIVE_SERVER2_TRANSPORT_MODE))) { + props.setProperty("transportMode", "http"); + } else { + return; + } + props.setProperty("httpPath", HiveConf.getVar(conf, ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH)); + } + + private void addHosts(Properties props) throws BeelineHS2ConnectionFileParseException { + // if zk HA is enabled get hosts property + if (HiveConf.getBoolVar(conf, + HiveConf.ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) { + addZKServiceDiscoveryHosts(props); + } else { + addDefaultHS2Hosts(props); + } + } + + private void addZKServiceDiscoveryHosts(Properties props) + throws BeelineHS2ConnectionFileParseException { + props.setProperty("serviceDiscoveryMode", "zooKeeper"); + props.setProperty("zooKeeperNamespace", + HiveConf.getVar(conf, ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE)); + props.setProperty("hosts", HiveConf.getVar(conf, ConfVars.HIVE_ZOOKEEPER_QUORUM)); + } + + private void addDefaultHS2Hosts(Properties props) throws BeelineHS2ConnectionFileParseException { + String hiveHost = System.getenv("HIVE_SERVER2_THRIFT_BIND_HOST"); + if (hiveHost == null) { + hiveHost = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST); + } + + InetAddress serverIPAddress; + try { + serverIPAddress = ServerUtils.getHostAddress(hiveHost); + } catch (UnknownHostException e) { + throw new BeelineHS2ConnectionFileParseException(e.getMessage(), e); + } + int portNum = getPortNum( + "http".equalsIgnoreCase(HiveConf.getVar(conf, ConfVars.HIVE_SERVER2_TRANSPORT_MODE))); + props.setProperty("hosts", serverIPAddress.getHostName() + ":" + portNum); + } + + private int getPortNum(boolean isHttp) { + String portString; + int portNum; + if (isHttp) { + portString = System.getenv("HIVE_SERVER2_THRIFT_HTTP_PORT"); + if (portString != null) { + portNum = Integer.parseInt(portString); + } else { + portNum = HiveConf.getIntVar(conf, ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT); + } + } else { + portString = System.getenv("HIVE_SERVER2_THRIFT_PORT"); + if (portString != null) { + portNum = Integer.parseInt(portString); + } else { + portNum = HiveConf.getIntVar(conf, ConfVars.HIVE_SERVER2_THRIFT_PORT); + } + } + return portNum; + } + + @Override + public String getFileLocation() { + return hiveSiteURI == null ? null : hiveSiteURI.toString(); + } +} diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java new file mode 100644 index 0000000000000000000000000000000000000000..004620400527cf89a3770b991d79c04e8ca1c4f8 --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java @@ -0,0 +1,118 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.io.File; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; + +/** + * This class implements HS2ConnectionFileParser for the user-specific connection configuration + * file named beeline-hs2-connection.xml. The class looks for this file in + * ${user.home}/.beeline, ${HIVE_CONF_DIR} or /etc/conf/hive in that order and uses the first file + * found in the above locations. + */ +public class UserHS2ConnectionFileParser implements HS2ConnectionFileParser { + + public static final String DEFAULT_CONNECTION_CONFIG_FILE_NAME = "beeline-hs2-connection.xml"; + public static final String DEFAULT_BEELINE_USER_CONF_LOCATION = + System.getProperty("user.home") + File.separator + + (System.getProperty("os.name").toLowerCase().indexOf("windows") != -1 ? "" : ".") + + "beeline" + File.separator; + public static final String ETC_HIVE_CONF_LOCATION = + File.separator + "etc" + File.separator + "conf" + File.separator + "hive"; + + private final List locations = new ArrayList<>(); + private static final Logger log = LoggerFactory.getLogger(UserHS2ConnectionFileParser.class); + + public UserHS2ConnectionFileParser() { + // file locations to be searched in the correct order + locations.add(DEFAULT_BEELINE_USER_CONF_LOCATION + DEFAULT_CONNECTION_CONFIG_FILE_NAME); + if (System.getenv("HIVE_CONF_DIR") != null) { + locations.add( + System.getenv("HIVE_CONF_DIR") + File.separator + DEFAULT_CONNECTION_CONFIG_FILE_NAME); + } + locations.add(ETC_HIVE_CONF_LOCATION + DEFAULT_CONNECTION_CONFIG_FILE_NAME); + } + + @VisibleForTesting + public UserHS2ConnectionFileParser(List testLocations) { + if(testLocations == null) { + return; + } + locations.addAll(testLocations); + } + + @Override + public Properties getConnectionProperties() throws BeelineHS2ConnectionFileParseException { + Properties props = new Properties(); + // get the file location, it not found return null nothing to do + String fileLocation = getFileLocation(); + if (fileLocation == null) { + log.debug("User connection configuration file not found"); + return props; + } + log.info("Using connection configuration file at " + fileLocation); + props.setProperty("url_prefix", "jdbc:hive2://"); + // load the properties from config file + Configuration conf = new Configuration(false); + conf.addResource(new Path(new File(fileLocation).toURI())); + Iterator> it; + try { + it = conf.iterator(); + } catch (Exception ex) { + throw new BeelineHS2ConnectionFileParseException(ex.getMessage(), ex); + } + while (it.hasNext()) { + Entry kv = it.next(); + String key = kv.getKey(); + if (key.startsWith(BEELINE_CONNECTION_PROPERTY_PREFIX)) { + props.setProperty(key.substring(BEELINE_CONNECTION_PROPERTY_PREFIX.length()), + kv.getValue()); + } else { + log.warn("Ignoring " + key + " since it does not start with " + + BEELINE_CONNECTION_PROPERTY_PREFIX); + } + } + return props; + } + + /* + * This method looks in locations specified above and returns the first location where the file + * exists. If the file does not exist in any one of the locations it returns null + */ + @Override + public String getFileLocation() { + for (String location : locations) { + if (new File(location).exists()) { + return location; + } + } + return null; + } +} \ No newline at end of file diff --git a/beeline/src/test/org/apache/hive/beeline/TestUserHS2ConnectionFileParser.java b/beeline/src/test/org/apache/hive/beeline/TestUserHS2ConnectionFileParser.java new file mode 100644 index 0000000000000000000000000000000000000000..df52eb3d1f77c38515a5fcf09f50fbf1cf3bd2ff --- /dev/null +++ b/beeline/src/test/org/apache/hive/beeline/TestUserHS2ConnectionFileParser.java @@ -0,0 +1,211 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hive.beeline.hs2connection.BeelineHS2ConnectionFileParseException; +import org.apache.hive.beeline.hs2connection.UserHS2ConnectionFileParser; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileUtils; +import org.apache.hive.common.util.HiveTestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +public class TestUserHS2ConnectionFileParser { + private final String LOCATION_1 = System.getProperty("java.io.tmpdir") + "loc1" + File.separator + + UserHS2ConnectionFileParser.DEFAULT_CONNECTION_CONFIG_FILE_NAME; + + private final String LOCATION_2 = System.getProperty("java.io.tmpdir") + "loc2" + File.separator + + UserHS2ConnectionFileParser.DEFAULT_CONNECTION_CONFIG_FILE_NAME; + + private final String LOCATION_3 = System.getProperty("java.io.tmpdir") + "loc3" + File.separator + + UserHS2ConnectionFileParser.DEFAULT_CONNECTION_CONFIG_FILE_NAME; + + List testLocations = new ArrayList<>(); + + @After + public void cleanUp() { + try { + deleteFile(LOCATION_1); + deleteFile(LOCATION_2); + deleteFile(LOCATION_3); + } catch (Exception e) { + e.printStackTrace(); + } + testLocations.clear(); + } + + @Test + public void testParseNoAuthentication() throws BeelineHS2ConnectionFileParseException { + String url = getParsedUrlFromConfigFile("test-hs2-connection-config-noauth.xml"); + String expectedUrl = "jdbc:hive2://localhost:10000/default;user=hive"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testParseZookeeper() throws BeelineHS2ConnectionFileParseException { + String url = getParsedUrlFromConfigFile("test-hs2-connection-zookeeper-config.xml"); + String expectedUrl = + "jdbc:hive2://zk-node-1:10000,zk-node-2:10001,zk-node-3:10004/default;serviceDiscoveryMode=zookeeper;zooKeeperNamespace=hiveserver2"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testParseWithKerberosNoSSL() throws BeelineHS2ConnectionFileParseException { + String url = getParsedUrlFromConfigFile("test-hs2-conn-conf-kerberos-nossl.xml"); + String expectedUrl = + "jdbc:hive2://localhost:10000/default;principal=hive/dummy-hostname@domain.com;ssl=false"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testParseWithKerberosSSL() throws BeelineHS2ConnectionFileParseException { + String url = getParsedUrlFromConfigFile("test-hs2-conn-conf-kerberos-ssl.xml"); + String expectedUrl = + "jdbc:hive2://localhost:10000/default;principal=hive/dummy-hostname@domain.com;ssl=true;" + + "sslTrustStore=test/truststore;trustStorePassword=testTruststorePassword"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testParseWithSSLAndHttpMode() throws BeelineHS2ConnectionFileParseException { + String url = getParsedUrlFromConfigFile("test-hs2-conn-conf-kerberos-http.xml"); + String expectedUrl = + "jdbc:hive2://localhost:10000/default;httpPath=testHTTPPath;principal=hive/dummy-hostname@domain.com;" + + "ssl=true;sslTrustStore=test/truststore;transportMode=http;trustStorePassword=testTruststorePassword"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testUrlWithHiveConfValues() throws Exception { + String url = getParsedUrlFromConfigFile("test-hs2-connection-conf-list.xml"); + String expectedUrl = + "jdbc:hive2://localhost:10000/default;user=hive?hive.cli.print.current.db=false#testVarName1=value1"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testUrlWithMultipleHiveConfValues() throws Exception { + String url = getParsedUrlFromConfigFile("test-hs2-connection-multi-conf-list.xml"); + String expectedUrl = + "jdbc:hive2://localhost:10000/default;user=hive?hive.cli.print.current.db=true;" + + "hive.cli.print.header=true#testVarName1=value1;testVarName2=value2"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + /* + * Tests if null value returned when file is not present in any of the lookup locations + */ + @Test + public void testNoLocationFoundCase() throws Exception { + testLocations.add(LOCATION_1); + testLocations.add(LOCATION_2); + testLocations.add(LOCATION_3); + UserHS2ConnectionFileParser testHS2ConfigManager = + new UserHS2ConnectionFileParser(testLocations); + Assert.assertTrue(testHS2ConfigManager.getConnectionProperties().isEmpty()); + } + + /* + * Tests if LOCATION_1 is returned when file is present in the first directory in lookup order + */ + @Test + public void testGetLocation1() throws Exception { + createNewFile(LOCATION_1); + testLocations.add(LOCATION_1); + testLocations.add(LOCATION_2); + testLocations.add(LOCATION_3); + UserHS2ConnectionFileParser testHS2ConfigManager = + new UserHS2ConnectionFileParser(testLocations); + Assert.assertTrue("File location " + LOCATION_1 + " was not returned", + LOCATION_1.equals(testHS2ConfigManager.getFileLocation())); + } + + /* + * Tests if LOCATION_3 is returned when the first file is found is later in lookup order + */ + @Test + public void testGetLocation3() throws Exception { + createNewFile(LOCATION_3); + testLocations.add(LOCATION_1); + testLocations.add(LOCATION_2); + testLocations.add(LOCATION_3); + UserHS2ConnectionFileParser testHS2ConfigManager = + new UserHS2ConnectionFileParser(testLocations); + Assert.assertTrue("File location " + LOCATION_3 + " was not returned", + LOCATION_3.equals(testHS2ConfigManager.getFileLocation())); + } + + /* + * Tests if it returns the first file present in the lookup order when files are present in the + * lookup order + */ + @Test + public void testGetLocationOrder() throws Exception { + createNewFile(LOCATION_2); + createNewFile(LOCATION_3); + testLocations.add(LOCATION_1); + testLocations.add(LOCATION_2); + testLocations.add(LOCATION_3); + UserHS2ConnectionFileParser testHS2ConfigManager = + new UserHS2ConnectionFileParser(testLocations); + Assert.assertTrue("File location " + LOCATION_2 + " was not returned", + LOCATION_2.equals(testHS2ConfigManager.getFileLocation())); + } + + private String getParsedUrlFromConfigFile(String filename) + throws BeelineHS2ConnectionFileParseException { + String path = HiveTestUtils.getFileFromClasspath(filename); + testLocations.add(path); + UserHS2ConnectionFileParser testHS2ConfigManager = + new UserHS2ConnectionFileParser(testLocations); + + String url = HS2ConnectionFileUtils.getUrl(testHS2ConfigManager.getConnectionProperties()); + return url; + } + + private void createNewFile(final String path) throws Exception { + File file = new File(path); + if (file.exists()) { + return; + } + String dir = path.substring(0, + path.indexOf(UserHS2ConnectionFileParser.DEFAULT_CONNECTION_CONFIG_FILE_NAME)); + if (!new File(dir).exists()) { + if (!new File(dir).mkdirs()) { + throw new Exception("Could not create directory " + dir); + } + } + if (!file.createNewFile()) { + throw new Exception("Could not create new file at " + path); + } + } + + private void deleteFile(final String path) throws Exception { + File file = new File(path); + if (file.exists()) { + if (!file.delete()) { + throw new Exception("Could not delete file at " + path); + } + } + } + +} diff --git a/beeline/src/test/resources/hive-site.xml b/beeline/src/test/resources/hive-site.xml index 5f310d68245275ac9dc24df45579784019eea332..fdda94bba292123cfdb1cf75b09cfca2408283b9 100644 --- a/beeline/src/test/resources/hive-site.xml +++ b/beeline/src/test/resources/hive-site.xml @@ -45,4 +45,9 @@ ${test.tmp.dir}/warehouse + + test.data.files + ${hive.root}/data/files + + diff --git a/beeline/src/test/resources/test-hs2-conn-conf-kerberos-http.xml b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-http.xml new file mode 100644 index 0000000000000000000000000000000000000000..75f4fb803885287f528c64214ab876c97bc34173 --- /dev/null +++ b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-http.xml @@ -0,0 +1,48 @@ + + + + + + beeline.hs2.connection.hosts + localhost:10000 + + + beeline.hs2.connection.principal + hive/dummy-hostname@domain.com + + + beeline.hs2.connection.ssl + true + + + beeline.hs2.connection.sslTrustStore + test/truststore + + + beeline.hs2.connection.trustStorePassword + testTruststorePassword + + + beeline.hs2.connection.transportMode + http + + + beeline.hs2.connection.httpPath + testHTTPPath + + \ No newline at end of file diff --git a/beeline/src/test/resources/test-hs2-conn-conf-kerberos-nossl.xml b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-nossl.xml new file mode 100644 index 0000000000000000000000000000000000000000..dbceee5c9d4575e4677113101e93b5d90f16f0b0 --- /dev/null +++ b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-nossl.xml @@ -0,0 +1,32 @@ + + + + + + beeline.hs2.connection.hosts + localhost:10000 + + + beeline.hs2.connection.principal + hive/dummy-hostname@domain.com + + + beeline.hs2.connection.ssl + false + + \ No newline at end of file diff --git a/beeline/src/test/resources/test-hs2-conn-conf-kerberos-ssl.xml b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-ssl.xml new file mode 100644 index 0000000000000000000000000000000000000000..7dce56c8d4753130d8f627aa3362245eac0e8a27 --- /dev/null +++ b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-ssl.xml @@ -0,0 +1,40 @@ + + + + + + beeline.hs2.connection.hosts + localhost:10000 + + + beeline.hs2.connection.principal + hive/dummy-hostname@domain.com + + + beeline.hs2.connection.ssl + true + + + beeline.hs2.connection.sslTrustStore + test/truststore + + + beeline.hs2.connection.trustStorePassword + testTruststorePassword + + \ No newline at end of file diff --git a/beeline/src/test/resources/test-hs2-connection-conf-list.xml b/beeline/src/test/resources/test-hs2-connection-conf-list.xml new file mode 100644 index 0000000000000000000000000000000000000000..6c022b1e36f86617a28e438261d1b0e2e16823bf --- /dev/null +++ b/beeline/src/test/resources/test-hs2-connection-conf-list.xml @@ -0,0 +1,36 @@ + + + + + + beeline.hs2.connection.hosts + localhost:10000 + + + beeline.hs2.connection.user + hive + + + beeline.hs2.connection.hiveconf + hive.cli.print.current.db=false + + + beeline.hs2.connection.hivevar + testVarName1=value1 + + diff --git a/beeline/src/test/resources/test-hs2-connection-config-noauth.xml b/beeline/src/test/resources/test-hs2-connection-config-noauth.xml new file mode 100644 index 0000000000000000000000000000000000000000..7a858e041c7960783d0d0f57a791d98da6b0ca0d --- /dev/null +++ b/beeline/src/test/resources/test-hs2-connection-config-noauth.xml @@ -0,0 +1,28 @@ + + + + + + beeline.hs2.connection.hosts + localhost:10000 + + + beeline.hs2.connection.user + hive + + \ No newline at end of file diff --git a/beeline/src/test/resources/test-hs2-connection-multi-conf-list.xml b/beeline/src/test/resources/test-hs2-connection-multi-conf-list.xml new file mode 100644 index 0000000000000000000000000000000000000000..7faae44baa34d9430aca754dc5bc58fcf46833ae --- /dev/null +++ b/beeline/src/test/resources/test-hs2-connection-multi-conf-list.xml @@ -0,0 +1,37 @@ + + + + + + beeline.hs2.connection.hosts + localhost:10000 + + + beeline.hs2.connection.user + hive + + + beeline.hs2.connection.hiveconf + hive.cli.print.current.db=true, hive.cli.print.header=true + + + beeline.hs2.connection.hivevar + testVarName1=value1, testVarName2=value2 + + + diff --git a/beeline/src/test/resources/test-hs2-connection-zookeeper-config.xml b/beeline/src/test/resources/test-hs2-connection-zookeeper-config.xml new file mode 100644 index 0000000000000000000000000000000000000000..aa95b76cd59c08637d7f0323d28064b18d636aa4 --- /dev/null +++ b/beeline/src/test/resources/test-hs2-connection-zookeeper-config.xml @@ -0,0 +1,32 @@ + + + + + + beeline.hs2.connection.hosts + zk-node-1:10000,zk-node-2:10001,zk-node-3:10004 + + + beeline.hs2.connection.serviceDiscoveryMode + zookeeper + + + beeline.hs2.connection.zooKeeperNamespace + hiveserver2 + + \ No newline at end of file diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connectionfile/TestBeelineConnectionUsingHiveSite.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connectionfile/TestBeelineConnectionUsingHiveSite.java new file mode 100644 index 0000000000000000000000000000000000000000..1716c9b8edc2adf419c3052584d3ca5c4c8eb422 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connectionfile/TestBeelineConnectionUsingHiveSite.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connectionfile; + +import java.io.File; + +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileParser; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.junit.Test; + +public class TestBeelineConnectionUsingHiveSite extends TestBeelineWithHS2ConnectionFile { + @Test + public void testBeelineConnectionHttp() throws Exception { + setupHs2(); + String path = createDefaultHs2ConnectionFile(); + testBeeLineConnection(path); + } + + @Test + public void testBeelineConnectionSSL() throws Exception { + setupSSLHs2(); + String path = createDefaultHs2ConnectionFile(); + testBeeLineConnection(path); + } + + @Test + public void testBeelineConnectionNoAuth() throws Exception { + setupNoAuthHs2(); + String path = createDefaultHs2ConnectionFile(); + testBeeLineConnection(path); + } + + /* + * tests if the beeline behaves like default mode if there is no user-specific connection + * configuration file + */ + @Test + public void testBeelineWithNoConnectionFile() throws Exception { + setupNoAuthHs2(); + testBeelineNoConnection(); + } + + @Test + public void testBeelineUsingArgs() throws Exception { + setupNoAuthHs2(); + String url = miniHS2.getBaseJdbcURL() + "default"; + String args[] = new String[] { "-u", url, "-n", System.getProperty("user.name"), "-p", "foo", + "-e", "show tables;" }; + testBeelineConnectionUsingCommandLine(args); + } + + private void setupNoAuthHs2() throws Exception { + // use default configuration for no-auth mode + miniHS2.start(confOverlay); + createTable(); + } + + private void setupSSLHs2() throws Exception { + confOverlay.put(ConfVars.HIVE_SERVER2_USE_SSL.varname, "true"); + confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname, + dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME); + confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname, + KEY_STORE_TRUST_STORE_PASSWORD); + miniHS2.start(confOverlay); + createTable(); + System.setProperty(JAVA_TRUST_STORE_PROP, dataFileDir + File.separator + TRUST_STORE_NAME); + System.setProperty(JAVA_TRUST_STORE_PASS_PROP, KEY_STORE_TRUST_STORE_PASSWORD); + } + + @Override + protected MiniHS2 getNewMiniHS2() throws Exception { + return new MiniHS2(hiveConf, MiniHS2.MiniClusterType.DFS_ONLY, true); + } + + private void setupHs2() throws Exception { + confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_HTTP_MODE); + confOverlay.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, HS2_HTTP_ENDPOINT); + confOverlay.put(ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true"); + miniHS2.start(confOverlay); + createTable(); + } + + private String createDefaultHs2ConnectionFile() throws Exception { + Hs2ConnectionXmlConfigFileWriter writer = new Hs2ConnectionXmlConfigFileWriter(); + String baseJdbcURL = miniHS2.getBaseJdbcURL(); + System.out.println(baseJdbcURL); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "user", + System.getProperty("user.name")); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "password", "foo"); + writer.close(); + return writer.path(); + } +} diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connectionfile/TestBeelineWithHS2ConnectionFile.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connectionfile/TestBeelineWithHS2ConnectionFile.java new file mode 100644 index 0000000000000000000000000000000000000000..3ea454165bd6f7fe987d20222ad574fc34d31d53 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connectionfile/TestBeelineWithHS2ConnectionFile.java @@ -0,0 +1,228 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connectionfile; + +import static org.junit.Assert.assertFalse; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.sql.DriverManager; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.beeline.BeeLine; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileParser; +import org.apache.hive.beeline.hs2connection.HiveSiteHS2ConnectionFileParser; +import org.apache.hive.beeline.hs2connection.UserHS2ConnectionFileParser; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.apache.hive.service.cli.CLIServiceClient; +import org.apache.hive.service.cli.HiveSQLException; +import org.apache.hive.service.cli.OperationHandle; +import org.apache.hive.service.cli.RowSet; +import org.apache.hive.service.cli.SessionHandle; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; + +public abstract class TestBeelineWithHS2ConnectionFile { + protected MiniHS2 miniHS2; + protected HiveConf hiveConf = new HiveConf(); + private final String tableName = "testBeelineTable"; + protected String dataFileDir = hiveConf.get("test.data.files"); + protected static final String LOCALHOST_KEY_STORE_NAME = "keystore.jks"; + protected static final String TRUST_STORE_NAME = "truststore.jks"; + protected static final String KEY_STORE_TRUST_STORE_PASSWORD = "HiveJdbc"; + protected static final String HS2_HTTP_MODE = "http"; + protected static final String HS2_HTTP_ENDPOINT = "cliservice"; + private final String fileLocation = + System.getProperty("java.io.tmpdir") + "testHs2ConnectionConfig.properties"; + protected static final String JAVA_TRUST_STORE_PROP = "javax.net.ssl.trustStore"; + protected static final String JAVA_TRUST_STORE_PASS_PROP = "javax.net.ssl.trustStorePassword"; + + protected Map confOverlay = new HashMap<>(); + + protected class TestBeeLine extends BeeLine { + UserHS2ConnectionFileParser testHs2ConfigFileManager; + ByteArrayOutputStream os; + + TestBeeLine(List defaultHS2ConnectionFiles) { + testHs2ConfigFileManager = new UserHS2ConnectionFileParser(defaultHS2ConnectionFiles); + os = new ByteArrayOutputStream(); + PrintStream beelineOutputStream = new PrintStream(os); + setOutputStream(beelineOutputStream); + setErrorStream(beelineOutputStream); + } + + TestBeeLine() { + testHs2ConfigFileManager = new UserHS2ConnectionFileParser(null); + os = new ByteArrayOutputStream(); + PrintStream beelineOutputStream = new PrintStream(os); + setOutputStream(beelineOutputStream); + setErrorStream(beelineOutputStream); + } + + public String getOutput() throws UnsupportedEncodingException { + return os.toString("UTF8"); + } + + @Override + public HS2ConnectionFileParser getDefaultHS2ConnectionFileParser() { + return testHs2ConfigFileManager; + } + + @Override + public HS2ConnectionFileParser getHiveSiteHS2ConnectionFileParser() { + HiveSiteHS2ConnectionFileParser ret = new HiveSiteHS2ConnectionFileParser(); + ret.setHiveConf(miniHS2.getHiveConf()); + return ret; + } + } + + /* + * Wrapper class to write a HS2ConnectionConfig file + */ + protected class Hs2ConnectionXmlConfigFileWriter { + private final PrintWriter writer; + private final File file; + private final Configuration conf; + + protected Hs2ConnectionXmlConfigFileWriter() throws IOException { + file = new File(fileLocation); + conf = new Configuration(false); + try { + if (file.exists()) { + file.delete(); + } + file.createNewFile(); + writer = new PrintWriter(file.getAbsolutePath(), "UTF-8"); + } finally { + file.deleteOnExit(); + } + } + + protected void writeProperty(String key, String value) { + conf.set(key, value); + } + + protected String path() { + return file.getAbsolutePath(); + } + + protected void close() throws IOException { + try { + conf.writeXml(writer); + } finally { + writer.close(); + } + } + } + + @BeforeClass + public static void beforeTest() throws Exception { + Class.forName(MiniHS2.getJdbcDriverName()); + } + + @Before + public void before() throws Exception { + DriverManager.setLoginTimeout(0); + if (!System.getProperty("test.data.files", "").isEmpty()) { + dataFileDir = System.getProperty("test.data.files"); + } + dataFileDir = dataFileDir.replace('\\', '/').replace("c:", ""); + hiveConf = new HiveConf(); + miniHS2 = getNewMiniHS2(); + confOverlay = new HashMap(); + confOverlay.put(ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); + confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, "binary"); + } + + protected MiniHS2 getNewMiniHS2() throws Exception { + return new MiniHS2(hiveConf); + } + + @After + public void tearDown() throws Exception { + if (miniHS2 != null && miniHS2.isStarted()) { + miniHS2.stop(); + } + miniHS2 = null; + System.clearProperty(JAVA_TRUST_STORE_PROP); + System.clearProperty(JAVA_TRUST_STORE_PASS_PROP); + } + + protected void createTable() throws HiveSQLException { + CLIServiceClient serviceClient = miniHS2.getServiceClient(); + SessionHandle sessHandle = serviceClient.openSession("foo", "bar"); + serviceClient.executeStatement(sessHandle, "DROP TABLE IF EXISTS " + tableName, confOverlay); + serviceClient.executeStatement(sessHandle, "CREATE TABLE " + tableName + " (id INT)", + confOverlay); + OperationHandle opHandle = + serviceClient.executeStatement(sessHandle, "SHOW TABLES", confOverlay); + RowSet rowSet = serviceClient.fetchResults(opHandle); + assertFalse(rowSet.numRows() == 0); + } + + protected String testBeeLineConnection(String path) throws IOException { + List testLocations = new ArrayList<>(); + testLocations.add(path); + TestBeeLine beeLine = new TestBeeLine(testLocations); + String[] args = new String[] { "-e", "show tables;" }; + beeLine.begin(args, null); + String output = beeLine.getOutput(); + System.out.println(output); + beeLine.close(); + Assert.assertNotNull(output); + Assert.assertTrue(tableName + " not found in result", + output.toLowerCase().contains(tableName.toLowerCase())); + return output; + } + + protected void testBeelineNoConnection() throws Exception { + TestBeeLine beeLine = new TestBeeLine(); + String[] args = new String[] { "-e", "show tables;" }; + beeLine.begin(args, null); + String output = beeLine.getOutput(); + System.out.println(output); + beeLine.close(); + Assert.assertNotNull(output); + Assert.assertTrue("output should contain no current connection but it retured " + output, + output.toLowerCase().contains("no current connection")); + } + + protected void testBeelineConnectionUsingCommandLine(String[] args) throws Exception { + TestBeeLine beeLine = new TestBeeLine(); + System.out.println("vihang - " + args.toString()); + beeLine.begin(args, null); + String output = beeLine.getOutput(); + System.out.println(output); + beeLine.close(); + Assert.assertNotNull(output); + Assert.assertTrue(tableName + " not found in result", + output.toLowerCase().contains(tableName.toLowerCase())); + } +} diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connectionfile/TestBeelineWithUserHs2ConnectionFile.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connectionfile/TestBeelineWithUserHs2ConnectionFile.java new file mode 100644 index 0000000000000000000000000000000000000000..9037cc77e1a2d60b4d22ed9802a0889d67a21037 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connectionfile/TestBeelineWithUserHs2ConnectionFile.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connectionfile; + +import java.io.File; +import java.net.URI; + +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileParser; +import org.junit.Test; + +public class TestBeelineWithUserHs2ConnectionFile extends TestBeelineWithHS2ConnectionFile { + + @Test + public void testBeelineConnectionHttp() throws Exception { + setupHttpHs2(); + String path = createHttpHs2ConnectionFile(); + testBeeLineConnection(path); + } + + private void setupHttpHs2() throws Exception { + confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_HTTP_MODE); + confOverlay.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, HS2_HTTP_ENDPOINT); + confOverlay.put(ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true"); + miniHS2.start(confOverlay); + createTable(); + } + + private String createHttpHs2ConnectionFile() throws Exception { + Hs2ConnectionXmlConfigFileWriter writer = new Hs2ConnectionXmlConfigFileWriter(); + String baseJdbcURL = miniHS2.getBaseJdbcURL(); + + URI uri = new URI(baseJdbcURL.substring(5)); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "hosts", + uri.getHost() + ":" + uri.getPort()); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "user", + System.getProperty("user.name")); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "password", + "foo"); + writer.writeProperty( + HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "transportMode", + HS2_HTTP_MODE); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "httpPath", + HS2_HTTP_ENDPOINT); + writer.close(); + return writer.path(); + } + + @Test + public void testBeelineConnectionNoAuth() throws Exception { + setupNoAuthConfHS2(); + String path = createNoAuthHs2ConnectionFile(); + testBeeLineConnection(path); + } + + private void setupNoAuthConfHS2() throws Exception { + // use default configuration for no-auth mode + miniHS2.start(confOverlay); + createTable(); + } + + private String createNoAuthHs2ConnectionFile() throws Exception { + Hs2ConnectionXmlConfigFileWriter writer = new Hs2ConnectionXmlConfigFileWriter(); + String baseJdbcURL = miniHS2.getBaseJdbcURL(); + URI uri = new URI(baseJdbcURL.substring(5)); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "hosts", + uri.getHost() + ":" + uri.getPort()); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "user", + System.getProperty("user.name")); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "password", + "foo"); + writer.close(); + return writer.path(); + } + + @Test + public void testBeelineConnectionSSL() throws Exception { + setupSslHs2(); + String path = createSSLHs2ConnectionFile(); + testBeeLineConnection(path); + } + + private String createSSLHs2ConnectionFile() throws Exception { + Hs2ConnectionXmlConfigFileWriter writer = new Hs2ConnectionXmlConfigFileWriter(); + String baseJdbcURL = miniHS2.getBaseJdbcURL(); + URI uri = new URI(baseJdbcURL.substring(5)); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "hosts", + uri.getHost() + ":" + uri.getPort()); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "user", + System.getProperty("user.name")); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "password", + "foo"); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "ssl", + "true"); + writer.writeProperty( + HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "trustStorePassword", + KEY_STORE_TRUST_STORE_PASSWORD); + writer.writeProperty( + HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "sslTrustStore", + dataFileDir + File.separator + TRUST_STORE_NAME); + writer.close(); + return writer.path(); + } + + private void setupSslHs2() throws Exception { + confOverlay.put(ConfVars.HIVE_SERVER2_USE_SSL.varname, "true"); + confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname, + dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME); + confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname, + KEY_STORE_TRUST_STORE_PASSWORD); + miniHS2.start(confOverlay); + createTable(); + } +}