commit 5e17919b6f0850f83275e0b1cbd59b0418509ca2 Author: Vihang Karajgaonkar Date: Wed Jun 22 12:48:45 2016 -0700 HIVE-14063 : Add beeline connection configuration file to automatically connect to HS2 using beeline This change adds a new optional configuration file for Beeline. If this file is present at predefined locations, Beeline will attempt to create the connection url using the hive-site.xml found in classpath and another user-specific configuration file. Beeline then connects automatically using the url generated based on these configuration files. The main objective of the change is to provide user another way to connect to the HiveServer2 without providing the connection url everytime. The configuration file uses hadoop xml format so that we can support encryption/obfuscation using hadoop credential manager API in the future. Properties in the user-specific configuration file override the properties derived from hive-site.xml. Tested using newly added unit tests and itests in various Hiveserver2 configurations. diff --git a/beeline/src/java/org/apache/hive/beeline/BeeLine.java b/beeline/src/java/org/apache/hive/beeline/BeeLine.java index 79922d267eaf13216e8a8e4826c050d334266579..fdbe0a2c6c37688ab511710bd43517908996f158 100644 --- a/beeline/src/java/org/apache/hive/beeline/BeeLine.java +++ b/beeline/src/java/org/apache/hive/beeline/BeeLine.java @@ -28,6 +28,7 @@ import java.io.EOFException; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; @@ -93,8 +94,15 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hive.beeline.cli.CliOptionsProcessor; import org.apache.hive.common.util.ShutdownHookManager; +import org.apache.hive.beeline.hs2connection.BeelineHS2ConnectionFileParseException; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileUtils; +import org.apache.hive.beeline.hs2connection.UserHS2ConnectionFileParser; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileParser; +import org.apache.hive.beeline.hs2connection.HiveSiteHS2ConnectionFileParser; import org.apache.thrift.transport.TTransportException; +import com.google.common.annotations.VisibleForTesting; + import org.apache.hive.jdbc.Utils; import org.apache.hive.jdbc.Utils.JdbcConnectionParams; @@ -279,7 +287,6 @@ "org.apache.hadoop.hive.jdbc.HiveDriver", })); - static { try { Class.forName("jline.console.ConsoleReader"); @@ -728,6 +735,46 @@ int initArgs(String[] args) { return -1; } + boolean connSuccessful = connectUsingArgs(cl); + // checks if default hs2 connection configuration file is present + // and uses it to connect if found + // no-op if the file is not present + if(!connSuccessful && !exit) { + connSuccessful = defaultBeelineConnect(); + } + + int code = 0; + if (cl.getOptionValues('e') != null) { + commands = Arrays.asList(cl.getOptionValues('e')); + } + + if (!commands.isEmpty() && getOpts().getScriptFile() != null) { + error("The '-e' and '-f' options cannot be specified simultaneously"); + return 1; + } else if(!commands.isEmpty() && !connSuccessful) { + error("Cannot run commands specified using -e. No current connection"); + return 1; + } + if (!commands.isEmpty()) { + for (Iterator i = commands.iterator(); i.hasNext();) { + String command = i.next().toString(); + debug(loc("executing-command", command)); + if (!dispatch(command)) { + code++; + } + } + exit = true; // execute and exit + } + return code; + } + + + /* + * Connects using the command line arguments. There are two + * possible ways to connect here 1. using the cmd line arguments like -u + * or using !properties + */ + private boolean connectUsingArgs(CommandLine cl) { String driver = null, user = null, pass = null, url = null; String auth = null; @@ -735,7 +782,7 @@ int initArgs(String[] args) { if (cl.hasOption("help")) { usage(); getOpts().setHelpAsked(true); - return 0; + return true; } Properties hiveVars = cl.getOptionProperties("hivevar"); @@ -764,24 +811,7 @@ int initArgs(String[] args) { } getOpts().setInitFiles(cl.getOptionValues("i")); getOpts().setScriptFile(cl.getOptionValue("f")); - if (cl.getOptionValues('e') != null) { - commands = Arrays.asList(cl.getOptionValues('e')); - } - - if (!commands.isEmpty() && getOpts().getScriptFile() != null) { - System.err.println("The '-e' and '-f' options cannot be specified simultaneously"); - return 1; - } - // TODO: temporary disable this for easier debugging - /* - if (url == null) { - url = BEELINE_DEFAULT_JDBC_URL; - } - if (driver == null) { - driver = BEELINE_DEFAULT_JDBC_DRIVER; - } - */ if (url != null) { if (user == null) { @@ -795,38 +825,25 @@ int initArgs(String[] args) { String com = constructCmd(url, user, pass, driver, false); String comForDebug = constructCmd(url, user, pass, driver, true); debug("issuing: " + comForDebug); - dispatch(com); - } - - // load property file - String propertyFile = cl.getOptionValue("property-file"); - if (propertyFile != null) { - try { - this.consoleReader = new ConsoleReader(); - } catch (IOException e) { - handleException(e); - } - if (!dispatch("!properties " + propertyFile)) { - exit = true; - return 1; - } - } - - int code = 0; - if (!commands.isEmpty()) { - for (Iterator i = commands.iterator(); i.hasNext();) { - String command = i.next().toString(); - debug(loc("executing-command", command)); - if (!dispatch(command)) { - code++; + return dispatch(com); + } else { + // load property file + String propertyFile = cl.getOptionValue("property-file"); + if (propertyFile != null) { + try { + this.consoleReader = new ConsoleReader(); + } catch (IOException e) { + handleException(e); + } + if (!dispatch("!properties " + propertyFile)) { + exit = true; + return false; } } - exit = true; // execute and exit } - return code; + return false; } - private void setHiveConfVar(String key, String val) { getOpts().getHiveConfVariables().put(key, val); if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key) && "mr".equals(val)) { @@ -906,13 +923,74 @@ public int begin(String[] args, InputStream inputStream) throws IOException { } catch (Exception e) { // ignore } - ConsoleReader reader = getConsoleReader(inputStream); + ConsoleReader reader = initializeConsoleReader(inputStream); return execute(reader, false); } finally { close(); } } + /* + * Attempts to make a connection using default HS2 connection config file if available + * if there connection is not made return false + * + */ + private boolean defaultBeelineConnect() { + String url; + try { + initializeConsoleReader(null); + url = getDefaultConnectionUrl(); + if (url == null) { + debug("Default hs2 connection config file not found"); + return false; + } + } catch (BeelineHS2ConnectionFileParseException | IOException e) { + error(e); + return false; + } + return dispatch("!connect " + url); + } + + + private String getDefaultConnectionUrl() throws BeelineHS2ConnectionFileParseException { + HS2ConnectionFileParser userHS2ConnFileParser = getUserHS2ConnFileParser(); + if (!userHS2ConnFileParser.configExists()) { + // nothing to do if there is no user HS2 connection configuration file + return null; + } + // get the connection properties from user specific config file + Properties userConnectionProperties = userHS2ConnFileParser.getConnectionProperties(); + // load the HS2 connection url properties from hive-site.xml if it is present in the classpath + HS2ConnectionFileParser hiveSiteParser = getHiveSiteHS2ConnectionFileParser(); + Properties hiveSiteConnectionProperties = hiveSiteParser.getConnectionProperties(); + // add/override properties found from hive-site with user-specific properties + for (String key : userConnectionProperties.stringPropertyNames()) { + if (hiveSiteConnectionProperties.containsKey(key)) { + debug("Overriding connection url property " + key + + " from user connection configuration file"); + } + hiveSiteConnectionProperties.setProperty(key, userConnectionProperties.getProperty(key)); + } + // return the url based on the aggregated connection properties + return HS2ConnectionFileUtils.getUrl(hiveSiteConnectionProperties); + } + + /* + * Increased visibility of this method is only for providing better test coverage + */ + @VisibleForTesting + public HS2ConnectionFileParser getUserHS2ConnFileParser() { + return new UserHS2ConnectionFileParser(); + } + + /* + * Increased visibility of this method is only for providing better test coverage + */ + @VisibleForTesting + public HS2ConnectionFileParser getHiveSiteHS2ConnectionFileParser() { + return new HiveSiteHS2ConnectionFileParser(); + } + int runInit() { String initFiles[] = getOpts().getInitFiles(); if (initFiles != null && initFiles.length != 0) { @@ -972,7 +1050,7 @@ private int executeFile(String fileName) { } else { initStream = new FileInputStream(fileName); } - return execute(getConsoleReader(initStream), !getOpts().getForce()); + return execute(initializeConsoleReader(initStream), !getOpts().getForce()); } catch (Throwable t) { handleException(t); return ERRNO_OTHER; @@ -1017,7 +1095,7 @@ public void close() { commands.closeall(null); } - public ConsoleReader getConsoleReader(InputStream inputStream) throws IOException { + public ConsoleReader initializeConsoleReader(InputStream inputStream) throws IOException { if (inputStream != null) { // ### NOTE: fix for sf.net bug 879425. // Working around an issue in jline-2.1.2, see https://github.com/jline/jline/issues/10 diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java b/beeline/src/java/org/apache/hive/beeline/Commands.java index 039e3549209682b5ebdde937607b77c3ec094527..6c3e7f70c811ad81aa8f3ee2ec2e42d98da7d330 100644 --- a/beeline/src/java/org/apache/hive/beeline/Commands.java +++ b/beeline/src/java/org/apache/hive/beeline/Commands.java @@ -22,12 +22,6 @@ */ package org.apache.hive.beeline; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.conf.HiveVariableSource; -import org.apache.hadoop.hive.conf.SystemVariables; -import org.apache.hadoop.hive.conf.VariableSubstitution; -import org.apache.hadoop.io.IOUtils; - import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -46,8 +40,8 @@ import java.sql.Driver; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.sql.SQLWarning; +import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -60,6 +54,12 @@ import java.util.TreeSet; import org.apache.hadoop.hive.common.cli.ShellCmdExecutor; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveVariableSource; +import org.apache.hadoop.hive.conf.SystemVariables; +import org.apache.hadoop.hive.conf.VariableSubstitution; +import org.apache.hadoop.io.IOUtils; import org.apache.hive.jdbc.HiveStatement; import org.apache.hive.jdbc.Utils; import org.apache.hive.jdbc.Utils.JdbcConnectionParams; @@ -1037,9 +1037,7 @@ private boolean executeInternal(String sql, boolean call) { } public String handleMultiLineCmd(String line) throws IOException { - //When using -e, console reader is not initialized and command is a single line - while (beeLine.getConsoleReader() != null && !(line.trim().endsWith(";")) && beeLine.getOpts() - .isAllowMultiLineCommand()) { + while (isMultiLine(line) && beeLine.getOpts().isAllowMultiLineCommand()) { StringBuilder prompt = new StringBuilder(beeLine.getPrompt()); if (!beeLine.getOpts().isSilent()) { @@ -1049,7 +1047,6 @@ public String handleMultiLineCmd(String line) throws IOException { } } } - String extra; if (beeLine.getOpts().isSilent() && beeLine.getOpts().getScriptFile() != null) { extra = beeLine.getConsoleReader().readLine(null, jline.console.ConsoleReader.NULL_MASK); @@ -1067,6 +1064,26 @@ public String handleMultiLineCmd(String line) throws IOException { return line; } + //returns true if statement represented by line is + //not complete and needs additional reading from + //console. Used in handleMultiLineCmd method + //assumes line would never be null when this method is called + private boolean isMultiLine(String line) { + line = line.trim(); + if (line.endsWith(";")) { + return false; + } + if (beeLine.isComment(line)) { + return false; + } + // handles the case like line = show tables; --test comment + List cmds = getCmdList(line, false); + if(!cmds.isEmpty() && cmds.get(cmds.size()-1).trim().startsWith("--")) { + return false; + } + return true; + } + public boolean sql(String line, boolean entireLineAsCommand) { return execute(line, false, entireLineAsCommand); } @@ -1409,6 +1426,10 @@ public boolean connect(String line) throws Exception { value = Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_PASSWD); if (value != null) { props.setProperty(JdbcConnectionParams.AUTH_PASSWD, value); + } else { + //if the password is not provided, beeline assumes a empty string as + //password + props.setProperty(JdbcConnectionParams.AUTH_PASSWD, ""); } } @@ -1465,7 +1486,6 @@ private String getProperty(Properties props, String[] keys) { return null; } - public boolean connect(Properties props) throws IOException { String url = getProperty(props, new String[] { JdbcConnectionParams.PROPERTY_URL, @@ -1507,13 +1527,14 @@ public boolean connect(Properties props) throws IOException { beeLine.info("Connecting to " + url); if (Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_PRINCIPAL) == null) { + String urlForPrompt = url.substring(0, url.contains(";") ? url.indexOf(';') : url.length()); if (username == null) { - username = beeLine.getConsoleReader().readLine("Enter username for " + url + ": "); + username = beeLine.getConsoleReader().readLine("Enter username for " + urlForPrompt + ": "); } props.setProperty(JdbcConnectionParams.AUTH_USER, username); if (password == null) { - password = beeLine.getConsoleReader().readLine("Enter password for " + url + ": ", - new Character('*')); + password = beeLine.getConsoleReader().readLine("Enter password for " + urlForPrompt + ": ", + new Character('*')); } props.setProperty(JdbcConnectionParams.AUTH_PASSWD, password); } @@ -1539,7 +1560,6 @@ public boolean connect(Properties props) throws IOException { } } - public boolean rehash(String line) { try { if (!(beeLine.assertConnection())) { diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/BeelineHS2ConnectionFileParseException.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/BeelineHS2ConnectionFileParseException.java new file mode 100644 index 0000000000000000000000000000000000000000..6e6018631a11782130c1375fa1cd398f216f9c17 --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/BeelineHS2ConnectionFileParseException.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +public class BeelineHS2ConnectionFileParseException extends Exception { + private static final long serialVersionUID = -748635913718300617L; + + BeelineHS2ConnectionFileParseException(String msg, Exception e) { + super(msg, e); + } + + public BeelineHS2ConnectionFileParseException(String msg) { + super(msg); + } +} diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileParser.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileParser.java new file mode 100644 index 0000000000000000000000000000000000000000..7173ee8fb65a4248d066bc6290f0fe2e11c918af --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileParser.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.util.Properties; + +/** + * HS2ConnectionFileParser provides a interface to be used by Beeline to parse a configuration (file + * or otherwise) to return a Java Properties object which contain key value pairs to be used to construct + * connection URL automatically for the Beeline connection + */ +public interface HS2ConnectionFileParser { + /** + * prefix string used for the keys + */ + public static final String BEELINE_CONNECTION_PROPERTY_PREFIX = "beeline.hs2.connection."; + /** + * Property key used to provide the URL prefix for the connection URL + */ + public static final String URL_PREFIX_PROPERTY_KEY = "url_prefix"; + /** + * Property key used to provide the default database in the connection URL + */ + public static final String DEFAULT_DB_PROPERTY_KEY = "defaultDB"; + /** + * Property key used to provide the hosts in the connection URL + */ + public static final String HOST_PROPERTY_KEY = "hosts"; + /** + * Property key used to provide the hive configuration key value pairs in the connection URL + */ + public static final String HIVE_CONF_PROPERTY_KEY = "hiveconf"; + /** + * Property key used to provide the hive variables in the connection URL + */ + public static final String HIVE_VAR_PROPERTY_KEY = "hivevar"; + + /** + * Returns a Java properties object which contain the key value pairs which can be used in the + * Beeline connection URL

+ * The properties returned must include url_prefix and hosts

+ * Following are some examples of the URLs and returned properties object

+ *

    + *
  • jdbc:hive2://hs2-instance1.example.com:10000/default;user=hive;password=mypassword should + * return [ "url_prefix"="jdbc:hive2://", "hosts"="hs2-instance1.example.com:10000", + * "defaultDB"=default, "user"="hive", "password"="mypassword" ] + *
  • jdbc:hive2://zk-instance1:10001,zk-instance2:10002,zk-instance3:10003/default; + * serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2 should return [ + * "url_prefix"="jdbc:hive2://", + * "hosts"="zk-instance1:10001,zk-instance2:10002,zk-instance3:10003", "defaultDB"="default", + * "serviceDiscoveryMode"="zooKeeper", "zooKeeperNamespace"="hiveserver2" ] + * + *
  • If hive_conf_list and hive_var_list is present in the url it should return a comma separated + * key=value pairs for each of them

    + * For example :

    + * jdbc:hive2://hs2-instance1.example.com:10000/default;user=hive?hive.cli.print.currentdb=true; + * hive.cli.print.header=true#hivevar:mytbl=customers;hivevar:mycol=id it should return [ + * "url_prefix"="jdbc:hive2://", "hosts"="hs2-instance1.example.com:10000", "defaultDB"="default", + * "user"="hive", "hiveconf"="hive.cli.print.currentdb=true, hive.cli.print.header=true", + * "hivevar"="hivevar:mytb1=customers, hivevar:mycol=id" ] + *

+ * + * @return Properties object which contain connection URL properties for Beeline connection. Returns an empty properties + * object if the connection configuration is not found + * @throws BeelineHS2ConnectionFileParseException if there is invalid key with appropriate message + */ + Properties getConnectionProperties() throws BeelineHS2ConnectionFileParseException; + /** + * + * @return returns true if the configuration exists else returns false + */ + boolean configExists(); +} \ No newline at end of file diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileUtils.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..ae9c4df5b28c916bfa85695d5cf1fad82f5bba3b --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/HS2ConnectionFileUtils.java @@ -0,0 +1,119 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +public class HS2ConnectionFileUtils { + + public static String getUrl(Properties props) throws BeelineHS2ConnectionFileParseException { + if (props == null || props.isEmpty()) { + return null; + } + // use remove instead of get so that it is not parsed again + // in the for loop below + String urlPrefix = (String) props.remove(HS2ConnectionFileParser.URL_PREFIX_PROPERTY_KEY); + if (urlPrefix == null || urlPrefix.isEmpty()) { + throw new BeelineHS2ConnectionFileParseException("url_prefix parameter cannot be empty"); + } + + String hosts = (String) props.remove(HS2ConnectionFileParser.HOST_PROPERTY_KEY); + if (hosts == null || hosts.isEmpty()) { + throw new BeelineHS2ConnectionFileParseException("hosts parameter cannot be empty"); + } + String defaultDB = (String) props.remove(HS2ConnectionFileParser.DEFAULT_DB_PROPERTY_KEY); + if (defaultDB == null) { + defaultDB = "default"; + } + // collect the hiveConfList and HiveVarList separately so that they can be + // appended once all the session list are added to the url + String hiveConfProperties = ""; + if (props.containsKey(HS2ConnectionFileParser.HIVE_CONF_PROPERTY_KEY)) { + hiveConfProperties = extractHiveVariables( + (String) props.remove(HS2ConnectionFileParser.HIVE_CONF_PROPERTY_KEY), true); + } + + String hiveVarProperties = ""; + if (props.containsKey(HS2ConnectionFileParser.HIVE_VAR_PROPERTY_KEY)) { + hiveVarProperties = extractHiveVariables( + (String) props.remove(HS2ConnectionFileParser.HIVE_VAR_PROPERTY_KEY), false); + } + StringBuilder urlSb = new StringBuilder(); + urlSb.append(urlPrefix.trim()); + urlSb.append(hosts.trim()); + urlSb.append(File.separator); + urlSb.append(defaultDB.trim()); + List keys = new ArrayList<>(props.stringPropertyNames()); + // sorting the keys from the properties helps to create + // a deterministic url which is tested for various configuration in + // TestHS2ConnectionConfigFileManager + Collections.sort(keys); + for (String propertyName : keys) { + urlSb.append(";"); + urlSb.append(propertyName); + urlSb.append("="); + urlSb.append(props.getProperty(propertyName)); + } + if (!hiveConfProperties.isEmpty()) { + urlSb.append(hiveConfProperties.toString()); + } + if (!hiveVarProperties.isEmpty()) { + urlSb.append(hiveVarProperties.toString()); + } + return urlSb.toString(); + } + + private static String extractHiveVariables(String propertyValue, boolean isHiveConf) + throws BeelineHS2ConnectionFileParseException { + StringBuilder hivePropertiesList = new StringBuilder(); + String delimiter; + if (isHiveConf) { + delimiter = "?"; + } else { + delimiter = "#"; + } + hivePropertiesList.append(delimiter); + addPropertyValues(propertyValue, hivePropertiesList); + return hivePropertiesList.toString(); + } + + private static void addPropertyValues(String value, StringBuilder hivePropertiesList) + throws BeelineHS2ConnectionFileParseException { + // There could be multiple keyValuePairs separated by comma + String[] values = value.split(","); + boolean first = true; + for (String keyValuePair : values) { + String[] keyValue = keyValuePair.split("="); + if (keyValue.length != 2) { + throw new BeelineHS2ConnectionFileParseException("Unable to parse " + keyValuePair + + " in hs2 connection config file"); + } + if (!first) { + hivePropertiesList.append(";"); + } + first = false; + hivePropertiesList.append(keyValue[0].trim()); + hivePropertiesList.append("="); + hivePropertiesList.append(keyValue[1].trim()); + } + } +} diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/HiveSiteHS2ConnectionFileParser.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/HiveSiteHS2ConnectionFileParser.java new file mode 100644 index 0000000000000000000000000000000000000000..f1814fad93ddf4981908609a30f043e702852fea --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/HiveSiteHS2ConnectionFileParser.java @@ -0,0 +1,172 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.net.InetAddress; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.common.ServerUtils; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; + +/* + * Looks for a hive-site.xml from the classpath. If found this class parses the hive-site.xml + * to return a set of connection properties which can be used to construct the connection url + * for Beeline connection + */ +public class HiveSiteHS2ConnectionFileParser implements HS2ConnectionFileParser { + private Configuration conf; + private final URL hiveSiteURI; + private final static String TRUSTSTORE_PASS_PROP = "javax.net.ssl.trustStorePassword"; + private final static String TRUSTSTORE_PROP = "javax.net.ssl.trustStore"; + private static final Logger log = LoggerFactory.getLogger(HiveSiteHS2ConnectionFileParser.class); + + public HiveSiteHS2ConnectionFileParser() { + hiveSiteURI = HiveConf.getHiveSiteLocation(); + conf = new Configuration(); + if(hiveSiteURI == null) { + log.debug("hive-site.xml not found for constructing the connection URL"); + } else { + log.info("Using hive-site.xml at " + hiveSiteURI); + conf.addResource(hiveSiteURI); + } + } + + @VisibleForTesting + void setHiveConf(HiveConf hiveConf) { + this.conf = hiveConf; + } + + @Override + public Properties getConnectionProperties() throws BeelineHS2ConnectionFileParseException { + Properties props = new Properties(); + if(!configExists()) { + return props; + } + props.setProperty(HS2ConnectionFileParser.URL_PREFIX_PROPERTY_KEY, "jdbc:hive2://"); + addHosts(props); + addSSL(props); + addKerberos(props); + addHttp(props); + return props; + } + + private void addSSL(Properties props) { + if (!HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_SERVER2_USE_SSL)) { + return; + } else { + props.setProperty("ssl", "true"); + } + String truststore = System.getenv(TRUSTSTORE_PROP); + if (truststore != null && truststore.isEmpty()) { + props.setProperty("sslTruststore", truststore); + } + String trustStorePassword = System.getenv(TRUSTSTORE_PASS_PROP); + if (trustStorePassword != null && !trustStorePassword.isEmpty()) { + props.setProperty("trustStorePassword", trustStorePassword); + } + String saslQop = HiveConf.getVar(conf, ConfVars.HIVE_SERVER2_THRIFT_SASL_QOP); + if (!"auth".equalsIgnoreCase(saslQop)) { + props.setProperty("sasl.qop", saslQop); + } + } + + private void addKerberos(Properties props) { + if ("KERBEROS".equals(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION))) { + props.setProperty("principal", + HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL)); + } + } + + private void addHttp(Properties props) { + if ("http".equalsIgnoreCase(HiveConf.getVar(conf, ConfVars.HIVE_SERVER2_TRANSPORT_MODE))) { + props.setProperty("transportMode", "http"); + } else { + return; + } + props.setProperty("httpPath", HiveConf.getVar(conf, ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH)); + } + + private void addHosts(Properties props) throws BeelineHS2ConnectionFileParseException { + // if zk HA is enabled get hosts property + if (HiveConf.getBoolVar(conf, + HiveConf.ConfVars.HIVE_SERVER2_SUPPORT_DYNAMIC_SERVICE_DISCOVERY)) { + addZKServiceDiscoveryHosts(props); + } else { + addDefaultHS2Hosts(props); + } + } + + private void addZKServiceDiscoveryHosts(Properties props) + throws BeelineHS2ConnectionFileParseException { + props.setProperty("serviceDiscoveryMode", "zooKeeper"); + props.setProperty("zooKeeperNamespace", + HiveConf.getVar(conf, ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE)); + props.setProperty("hosts", HiveConf.getVar(conf, ConfVars.HIVE_ZOOKEEPER_QUORUM)); + } + + private void addDefaultHS2Hosts(Properties props) throws BeelineHS2ConnectionFileParseException { + String hiveHost = System.getenv("HIVE_SERVER2_THRIFT_BIND_HOST"); + if (hiveHost == null) { + hiveHost = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST); + } + + InetAddress serverIPAddress; + try { + serverIPAddress = ServerUtils.getHostAddress(hiveHost); + } catch (UnknownHostException e) { + throw new BeelineHS2ConnectionFileParseException(e.getMessage(), e); + } + int portNum = getPortNum( + "http".equalsIgnoreCase(HiveConf.getVar(conf, ConfVars.HIVE_SERVER2_TRANSPORT_MODE))); + props.setProperty("hosts", serverIPAddress.getHostName() + ":" + portNum); + } + + private int getPortNum(boolean isHttp) { + String portString; + int portNum; + if (isHttp) { + portString = System.getenv("HIVE_SERVER2_THRIFT_HTTP_PORT"); + if (portString != null) { + portNum = Integer.parseInt(portString); + } else { + portNum = HiveConf.getIntVar(conf, ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT); + } + } else { + portString = System.getenv("HIVE_SERVER2_THRIFT_PORT"); + if (portString != null) { + portNum = Integer.parseInt(portString); + } else { + portNum = HiveConf.getIntVar(conf, ConfVars.HIVE_SERVER2_THRIFT_PORT); + } + } + return portNum; + } + + @Override + public boolean configExists() { + return (hiveSiteURI != null); + } +} diff --git a/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java b/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java new file mode 100644 index 0000000000000000000000000000000000000000..93a6231cb4ae6efe5d49d29acb97b8c88471b341 --- /dev/null +++ b/beeline/src/java/org/apache/hive/beeline/hs2connection/UserHS2ConnectionFileParser.java @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.annotations.VisibleForTesting; + +/** + * This class implements HS2ConnectionFileParser for the user-specific connection configuration + * file named beeline-hs2-connection.xml. The class looks for this file in + * ${user.home}/.beeline, ${HIVE_CONF_DIR} or /etc/conf/hive in that order and uses the first file + * found in the above locations. + */ +public class UserHS2ConnectionFileParser implements HS2ConnectionFileParser { + + public static final String DEFAULT_CONNECTION_CONFIG_FILE_NAME = "beeline-hs2-connection.xml"; + public static final String DEFAULT_BEELINE_USER_CONF_LOCATION = + System.getProperty("user.home") + File.separator + + (System.getProperty("os.name").toLowerCase().indexOf("windows") != -1 ? "" : ".") + + "beeline" + File.separator; + public static final String ETC_HIVE_CONF_LOCATION = + File.separator + "etc" + File.separator + "conf" + File.separator + "hive"; + + private final List locations = new ArrayList<>(); + private static final Logger log = LoggerFactory.getLogger(UserHS2ConnectionFileParser.class); + + public UserHS2ConnectionFileParser() { + // file locations to be searched in the correct order + locations.add(DEFAULT_BEELINE_USER_CONF_LOCATION + DEFAULT_CONNECTION_CONFIG_FILE_NAME); + if (System.getenv("HIVE_CONF_DIR") != null) { + locations.add( + System.getenv("HIVE_CONF_DIR") + File.separator + DEFAULT_CONNECTION_CONFIG_FILE_NAME); + } + locations.add(ETC_HIVE_CONF_LOCATION + DEFAULT_CONNECTION_CONFIG_FILE_NAME); + } + + @VisibleForTesting + UserHS2ConnectionFileParser(List testLocations) { + if(testLocations == null) { + return; + } + locations.addAll(testLocations); + } + + @Override + public Properties getConnectionProperties() throws BeelineHS2ConnectionFileParseException { + Properties props = new Properties(); + String fileLocation = getFileLocation(); + if (fileLocation == null) { + log.debug("User connection configuration file not found"); + return props; + } + log.info("Using connection configuration file at " + fileLocation); + props.setProperty(HS2ConnectionFileParser.URL_PREFIX_PROPERTY_KEY, "jdbc:hive2://"); + // load the properties from config file + Configuration conf = new Configuration(false); + conf.addResource(new Path(new File(fileLocation).toURI())); + try { + for (Entry kv : conf) { + String key = kv.getKey(); + if (key.startsWith(BEELINE_CONNECTION_PROPERTY_PREFIX)) { + props.setProperty(key.substring(BEELINE_CONNECTION_PROPERTY_PREFIX.length()), + kv.getValue()); + } else { + log.warn("Ignoring " + key + " since it does not start with " + + BEELINE_CONNECTION_PROPERTY_PREFIX); + } + } + } catch (Exception ex) { + throw new BeelineHS2ConnectionFileParseException(ex.getMessage(), ex); + } + + return props; + } + + @Override + public boolean configExists() { + return (getFileLocation() != null); + } + /* + * This method looks in locations specified above and returns the first location where the file + * exists. If the file does not exist in any one of the locations it returns null + */ + String getFileLocation() { + for (String location : locations) { + if (new File(location).exists()) { + return location; + } + } + return null; + } +} \ No newline at end of file diff --git a/beeline/src/test/org/apache/hive/beeline/TestBeeLineHistory.java b/beeline/src/test/org/apache/hive/beeline/TestBeeLineHistory.java index e86197626a76474c189b683fa55f02959c7a8bff..489db56a82c70d02fb78f6483c9c94da54a30560 100644 --- a/beeline/src/test/org/apache/hive/beeline/TestBeeLineHistory.java +++ b/beeline/src/test/org/apache/hive/beeline/TestBeeLineHistory.java @@ -58,7 +58,7 @@ public void testNumHistories() throws Exception { BeeLine beeline = new BeeLine(); beeline.getOpts().setHistoryFile(fileName); beeline.setOutputStream(ops); - beeline.getConsoleReader(null); + beeline.initializeConsoleReader(null); beeline.dispatch("!history"); String output = os.toString("UTF-8"); int numHistories = output.split("\n").length; @@ -73,7 +73,7 @@ public void testHistory() throws Exception { BeeLine beeline = new BeeLine(); beeline.getOpts().setHistoryFile(fileName); beeline.setOutputStream(ops); - beeline.getConsoleReader(null); + beeline.initializeConsoleReader(null); beeline.dispatch("!history"); String output = os.toString("UTF-8"); String[] tmp = output.split("\n"); diff --git a/beeline/src/test/org/apache/hive/beeline/hs2connection/TestUserHS2ConnectionFileParser.java b/beeline/src/test/org/apache/hive/beeline/hs2connection/TestUserHS2ConnectionFileParser.java new file mode 100644 index 0000000000000000000000000000000000000000..05b2af43d3640093577868ef4f0e56a3a2ac1d34 --- /dev/null +++ b/beeline/src/test/org/apache/hive/beeline/hs2connection/TestUserHS2ConnectionFileParser.java @@ -0,0 +1,211 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + +import org.apache.hive.beeline.hs2connection.BeelineHS2ConnectionFileParseException; +import org.apache.hive.beeline.hs2connection.UserHS2ConnectionFileParser; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileUtils; +import org.apache.hive.common.util.HiveTestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +public class TestUserHS2ConnectionFileParser { + private final String LOCATION_1 = System.getProperty("java.io.tmpdir") + "loc1" + File.separator + + UserHS2ConnectionFileParser.DEFAULT_CONNECTION_CONFIG_FILE_NAME; + + private final String LOCATION_2 = System.getProperty("java.io.tmpdir") + "loc2" + File.separator + + UserHS2ConnectionFileParser.DEFAULT_CONNECTION_CONFIG_FILE_NAME; + + private final String LOCATION_3 = System.getProperty("java.io.tmpdir") + "loc3" + File.separator + + UserHS2ConnectionFileParser.DEFAULT_CONNECTION_CONFIG_FILE_NAME; + + List testLocations = new ArrayList<>(); + + @After + public void cleanUp() { + try { + deleteFile(LOCATION_1); + deleteFile(LOCATION_2); + deleteFile(LOCATION_3); + } catch (Exception e) { + e.printStackTrace(); + } + testLocations.clear(); + } + + @Test + public void testParseNoAuthentication() throws BeelineHS2ConnectionFileParseException { + String url = getParsedUrlFromConfigFile("test-hs2-connection-config-noauth.xml"); + String expectedUrl = "jdbc:hive2://localhost:10000/default;user=hive"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testParseZookeeper() throws BeelineHS2ConnectionFileParseException { + String url = getParsedUrlFromConfigFile("test-hs2-connection-zookeeper-config.xml"); + String expectedUrl = + "jdbc:hive2://zk-node-1:10000,zk-node-2:10001,zk-node-3:10004/default;serviceDiscoveryMode=zookeeper;zooKeeperNamespace=hiveserver2"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testParseWithKerberosNoSSL() throws BeelineHS2ConnectionFileParseException { + String url = getParsedUrlFromConfigFile("test-hs2-conn-conf-kerberos-nossl.xml"); + String expectedUrl = + "jdbc:hive2://localhost:10000/default;principal=hive/dummy-hostname@domain.com;ssl=false"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testParseWithKerberosSSL() throws BeelineHS2ConnectionFileParseException { + String url = getParsedUrlFromConfigFile("test-hs2-conn-conf-kerberos-ssl.xml"); + String expectedUrl = + "jdbc:hive2://localhost:10000/default;principal=hive/dummy-hostname@domain.com;ssl=true;" + + "sslTrustStore=test/truststore;trustStorePassword=testTruststorePassword"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testParseWithSSLAndHttpMode() throws BeelineHS2ConnectionFileParseException { + String url = getParsedUrlFromConfigFile("test-hs2-conn-conf-kerberos-http.xml"); + String expectedUrl = + "jdbc:hive2://localhost:10000/default;httpPath=testHTTPPath;principal=hive/dummy-hostname@domain.com;" + + "ssl=true;sslTrustStore=test/truststore;transportMode=http;trustStorePassword=testTruststorePassword"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testUrlWithHiveConfValues() throws Exception { + String url = getParsedUrlFromConfigFile("test-hs2-connection-conf-list.xml"); + String expectedUrl = + "jdbc:hive2://localhost:10000/default;user=hive?hive.cli.print.current.db=false#testVarName1=value1"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + @Test + public void testUrlWithMultipleHiveConfValues() throws Exception { + String url = getParsedUrlFromConfigFile("test-hs2-connection-multi-conf-list.xml"); + String expectedUrl = + "jdbc:hive2://localhost:10000/default;user=hive?hive.cli.print.current.db=true;" + + "hive.cli.print.header=true#testVarName1=value1;testVarName2=value2"; + Assert.assertTrue("Expected " + expectedUrl + " got " + url, expectedUrl.equals(url)); + } + + /* + * Tests if null value returned when file is not present in any of the lookup locations + */ + @Test + public void testNoLocationFoundCase() throws Exception { + testLocations.add(LOCATION_1); + testLocations.add(LOCATION_2); + testLocations.add(LOCATION_3); + UserHS2ConnectionFileParser testHS2ConfigManager = + new UserHS2ConnectionFileParser(testLocations); + Assert.assertTrue(testHS2ConfigManager.getConnectionProperties().isEmpty()); + } + + /* + * Tests if LOCATION_1 is returned when file is present in the first directory in lookup order + */ + @Test + public void testGetLocation1() throws Exception { + createNewFile(LOCATION_1); + testLocations.add(LOCATION_1); + testLocations.add(LOCATION_2); + testLocations.add(LOCATION_3); + UserHS2ConnectionFileParser testHS2ConfigManager = + new UserHS2ConnectionFileParser(testLocations); + Assert.assertTrue("File location " + LOCATION_1 + " was not returned", + LOCATION_1.equals(testHS2ConfigManager.getFileLocation())); + } + + /* + * Tests if LOCATION_3 is returned when the first file is found is later in lookup order + */ + @Test + public void testGetLocation3() throws Exception { + createNewFile(LOCATION_3); + testLocations.add(LOCATION_1); + testLocations.add(LOCATION_2); + testLocations.add(LOCATION_3); + UserHS2ConnectionFileParser testHS2ConfigManager = + new UserHS2ConnectionFileParser(testLocations); + Assert.assertTrue("File location " + LOCATION_3 + " was not returned", + LOCATION_3.equals(testHS2ConfigManager.getFileLocation())); + } + + /* + * Tests if it returns the first file present in the lookup order when files are present in the + * lookup order + */ + @Test + public void testGetLocationOrder() throws Exception { + createNewFile(LOCATION_2); + createNewFile(LOCATION_3); + testLocations.add(LOCATION_1); + testLocations.add(LOCATION_2); + testLocations.add(LOCATION_3); + UserHS2ConnectionFileParser testHS2ConfigManager = + new UserHS2ConnectionFileParser(testLocations); + Assert.assertTrue("File location " + LOCATION_2 + " was not returned", + LOCATION_2.equals(testHS2ConfigManager.getFileLocation())); + } + + private String getParsedUrlFromConfigFile(String filename) + throws BeelineHS2ConnectionFileParseException { + String path = HiveTestUtils.getFileFromClasspath(filename); + testLocations.add(path); + UserHS2ConnectionFileParser testHS2ConfigManager = + new UserHS2ConnectionFileParser(testLocations); + + String url = HS2ConnectionFileUtils.getUrl(testHS2ConfigManager.getConnectionProperties()); + return url; + } + + private void createNewFile(final String path) throws Exception { + File file = new File(path); + if (file.exists()) { + return; + } + String dir = path.substring(0, + path.indexOf(UserHS2ConnectionFileParser.DEFAULT_CONNECTION_CONFIG_FILE_NAME)); + if (!new File(dir).exists()) { + if (!new File(dir).mkdirs()) { + throw new Exception("Could not create directory " + dir); + } + } + if (!file.createNewFile()) { + throw new Exception("Could not create new file at " + path); + } + } + + private void deleteFile(final String path) throws Exception { + File file = new File(path); + if (file.exists()) { + if (!file.delete()) { + throw new Exception("Could not delete file at " + path); + } + } + } + +} diff --git a/beeline/src/test/resources/hive-site.xml b/beeline/src/test/resources/hive-site.xml index 5f310d68245275ac9dc24df45579784019eea332..fdda94bba292123cfdb1cf75b09cfca2408283b9 100644 --- a/beeline/src/test/resources/hive-site.xml +++ b/beeline/src/test/resources/hive-site.xml @@ -45,4 +45,9 @@ ${test.tmp.dir}/warehouse + + test.data.files + ${hive.root}/data/files + + diff --git a/beeline/src/test/resources/test-hs2-conn-conf-kerberos-http.xml b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-http.xml new file mode 100644 index 0000000000000000000000000000000000000000..75f4fb803885287f528c64214ab876c97bc34173 --- /dev/null +++ b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-http.xml @@ -0,0 +1,48 @@ + + + + + + beeline.hs2.connection.hosts + localhost:10000 + + + beeline.hs2.connection.principal + hive/dummy-hostname@domain.com + + + beeline.hs2.connection.ssl + true + + + beeline.hs2.connection.sslTrustStore + test/truststore + + + beeline.hs2.connection.trustStorePassword + testTruststorePassword + + + beeline.hs2.connection.transportMode + http + + + beeline.hs2.connection.httpPath + testHTTPPath + + \ No newline at end of file diff --git a/beeline/src/test/resources/test-hs2-conn-conf-kerberos-nossl.xml b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-nossl.xml new file mode 100644 index 0000000000000000000000000000000000000000..dbceee5c9d4575e4677113101e93b5d90f16f0b0 --- /dev/null +++ b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-nossl.xml @@ -0,0 +1,32 @@ + + + + + + beeline.hs2.connection.hosts + localhost:10000 + + + beeline.hs2.connection.principal + hive/dummy-hostname@domain.com + + + beeline.hs2.connection.ssl + false + + \ No newline at end of file diff --git a/beeline/src/test/resources/test-hs2-conn-conf-kerberos-ssl.xml b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-ssl.xml new file mode 100644 index 0000000000000000000000000000000000000000..7dce56c8d4753130d8f627aa3362245eac0e8a27 --- /dev/null +++ b/beeline/src/test/resources/test-hs2-conn-conf-kerberos-ssl.xml @@ -0,0 +1,40 @@ + + + + + + beeline.hs2.connection.hosts + localhost:10000 + + + beeline.hs2.connection.principal + hive/dummy-hostname@domain.com + + + beeline.hs2.connection.ssl + true + + + beeline.hs2.connection.sslTrustStore + test/truststore + + + beeline.hs2.connection.trustStorePassword + testTruststorePassword + + \ No newline at end of file diff --git a/beeline/src/test/resources/test-hs2-connection-conf-list.xml b/beeline/src/test/resources/test-hs2-connection-conf-list.xml new file mode 100644 index 0000000000000000000000000000000000000000..6c022b1e36f86617a28e438261d1b0e2e16823bf --- /dev/null +++ b/beeline/src/test/resources/test-hs2-connection-conf-list.xml @@ -0,0 +1,36 @@ + + + + + + beeline.hs2.connection.hosts + localhost:10000 + + + beeline.hs2.connection.user + hive + + + beeline.hs2.connection.hiveconf + hive.cli.print.current.db=false + + + beeline.hs2.connection.hivevar + testVarName1=value1 + + diff --git a/beeline/src/test/resources/test-hs2-connection-config-noauth.xml b/beeline/src/test/resources/test-hs2-connection-config-noauth.xml new file mode 100644 index 0000000000000000000000000000000000000000..7a858e041c7960783d0d0f57a791d98da6b0ca0d --- /dev/null +++ b/beeline/src/test/resources/test-hs2-connection-config-noauth.xml @@ -0,0 +1,28 @@ + + + + + + beeline.hs2.connection.hosts + localhost:10000 + + + beeline.hs2.connection.user + hive + + \ No newline at end of file diff --git a/beeline/src/test/resources/test-hs2-connection-multi-conf-list.xml b/beeline/src/test/resources/test-hs2-connection-multi-conf-list.xml new file mode 100644 index 0000000000000000000000000000000000000000..7faae44baa34d9430aca754dc5bc58fcf46833ae --- /dev/null +++ b/beeline/src/test/resources/test-hs2-connection-multi-conf-list.xml @@ -0,0 +1,37 @@ + + + + + + beeline.hs2.connection.hosts + localhost:10000 + + + beeline.hs2.connection.user + hive + + + beeline.hs2.connection.hiveconf + hive.cli.print.current.db=true, hive.cli.print.header=true + + + beeline.hs2.connection.hivevar + testVarName1=value1, testVarName2=value2 + + + diff --git a/beeline/src/test/resources/test-hs2-connection-zookeeper-config.xml b/beeline/src/test/resources/test-hs2-connection-zookeeper-config.xml new file mode 100644 index 0000000000000000000000000000000000000000..aa95b76cd59c08637d7f0323d28064b18d636aa4 --- /dev/null +++ b/beeline/src/test/resources/test-hs2-connection-zookeeper-config.xml @@ -0,0 +1,32 @@ + + + + + + beeline.hs2.connection.hosts + zk-node-1:10000,zk-node-2:10001,zk-node-3:10004 + + + beeline.hs2.connection.serviceDiscoveryMode + zookeeper + + + beeline.hs2.connection.zooKeeperNamespace + hiveserver2 + + \ No newline at end of file diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java new file mode 100644 index 0000000000000000000000000000000000000000..fe776678530f3e0d44411051ae07b54979c1721a --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java @@ -0,0 +1,109 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.io.File; + +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileParser; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.junit.Test; + +public class TestBeelineConnectionUsingHiveSite extends TestBeelineWithHS2ConnectionFile { + @Test + public void testBeelineConnectionHttp() throws Exception { + setupHs2(); + String path = createDefaultHs2ConnectionFile(); + testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); + } + + @Test + public void testBeelineConnectionSSL() throws Exception { + setupSSLHs2(); + String path = createDefaultHs2ConnectionFile(); + testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); + } + + @Test + public void testBeelineConnectionNoAuth() throws Exception { + setupNoAuthHs2(); + String path = createDefaultHs2ConnectionFile(); + testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); + } + + /* + * tests if the beeline behaves like default mode if there is no user-specific connection + * configuration file + */ + @Test + public void testBeelineWithNoConnectionFile() throws Exception { + setupNoAuthHs2(); + testBeeLineConnection(null, new String[] { "-e", "show tables;" }, "no current connection"); + } + + @Test + public void testBeelineUsingArgs() throws Exception { + setupNoAuthHs2(); + String url = miniHS2.getBaseJdbcURL() + "default"; + String args[] = new String[] { "-u", url, "-n", System.getProperty("user.name"), "-p", "foo", + "-e", "show tables;" }; + testBeeLineConnection(null, args, tableName); + } + + private void setupNoAuthHs2() throws Exception { + // use default configuration for no-auth mode + miniHS2.start(confOverlay); + createTable(); + } + + private void setupSSLHs2() throws Exception { + confOverlay.put(ConfVars.HIVE_SERVER2_USE_SSL.varname, "true"); + confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname, + dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME); + confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname, + KEY_STORE_TRUST_STORE_PASSWORD); + miniHS2.start(confOverlay); + createTable(); + System.setProperty(JAVA_TRUST_STORE_PROP, dataFileDir + File.separator + TRUST_STORE_NAME); + System.setProperty(JAVA_TRUST_STORE_PASS_PROP, KEY_STORE_TRUST_STORE_PASSWORD); + } + + @Override + protected MiniHS2 getNewMiniHS2() throws Exception { + return new MiniHS2(hiveConf, MiniHS2.MiniClusterType.DFS_ONLY, true); + } + + private void setupHs2() throws Exception { + confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_HTTP_MODE); + confOverlay.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, HS2_HTTP_ENDPOINT); + confOverlay.put(ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true"); + miniHS2.start(confOverlay); + createTable(); + } + + private String createDefaultHs2ConnectionFile() throws Exception { + Hs2ConnectionXmlConfigFileWriter writer = new Hs2ConnectionXmlConfigFileWriter(); + String baseJdbcURL = miniHS2.getBaseJdbcURL(); + System.out.println(baseJdbcURL); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "user", + System.getProperty("user.name")); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "password", "foo"); + writer.close(); + return writer.path(); + } +} diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithHS2ConnectionFile.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithHS2ConnectionFile.java new file mode 100644 index 0000000000000000000000000000000000000000..32e9afd81d3aa8dc3969b50510d245d218f4fc24 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithHS2ConnectionFile.java @@ -0,0 +1,214 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import static org.junit.Assert.assertFalse; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.sql.DriverManager; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.beeline.BeeLine; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileParser; +import org.apache.hive.beeline.hs2connection.HiveSiteHS2ConnectionFileParser; +import org.apache.hive.beeline.hs2connection.UserHS2ConnectionFileParser; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.apache.hive.service.cli.CLIServiceClient; +import org.apache.hive.service.cli.HiveSQLException; +import org.apache.hive.service.cli.OperationHandle; +import org.apache.hive.service.cli.RowSet; +import org.apache.hive.service.cli.SessionHandle; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; + +public abstract class TestBeelineWithHS2ConnectionFile { + protected MiniHS2 miniHS2; + protected HiveConf hiveConf = new HiveConf(); + protected final String tableName = "testBeelineTable"; + protected String dataFileDir = hiveConf.get("test.data.files"); + protected static final String LOCALHOST_KEY_STORE_NAME = "keystore.jks"; + protected static final String TRUST_STORE_NAME = "truststore.jks"; + protected static final String KEY_STORE_TRUST_STORE_PASSWORD = "HiveJdbc"; + protected static final String HS2_HTTP_MODE = "http"; + protected static final String HS2_HTTP_ENDPOINT = "cliservice"; + private final String fileLocation = + System.getProperty("java.io.tmpdir") + "testHs2ConnectionConfig.xml"; + protected static final String JAVA_TRUST_STORE_PROP = "javax.net.ssl.trustStore"; + protected static final String JAVA_TRUST_STORE_PASS_PROP = "javax.net.ssl.trustStorePassword"; + + protected Map confOverlay = new HashMap<>(); + + protected class TestBeeLine extends BeeLine { + UserHS2ConnectionFileParser testHs2ConfigFileManager; + ByteArrayOutputStream os; + + TestBeeLine(List defaultHS2ConnectionFiles) { + testHs2ConfigFileManager = new UserHS2ConnectionFileParser(defaultHS2ConnectionFiles); + os = new ByteArrayOutputStream(); + PrintStream beelineOutputStream = new PrintStream(os); + setOutputStream(beelineOutputStream); + setErrorStream(beelineOutputStream); + } + + TestBeeLine() { + testHs2ConfigFileManager = new UserHS2ConnectionFileParser(null); + os = new ByteArrayOutputStream(); + PrintStream beelineOutputStream = new PrintStream(os); + setOutputStream(beelineOutputStream); + setErrorStream(beelineOutputStream); + } + + public String getOutput() throws UnsupportedEncodingException { + return os.toString("UTF8"); + } + + @Override + public HS2ConnectionFileParser getUserHS2ConnFileParser() { + return testHs2ConfigFileManager; + } + + @Override + public HS2ConnectionFileParser getHiveSiteHS2ConnectionFileParser() { + HiveSiteHS2ConnectionFileParser ret = new HiveSiteHS2ConnectionFileParser(); + ret.setHiveConf(miniHS2.getHiveConf()); + return ret; + } + } + + /* + * Wrapper class to write a HS2ConnectionConfig file + */ + protected class Hs2ConnectionXmlConfigFileWriter { + private final PrintWriter writer; + private final File file; + private final Configuration conf; + + protected Hs2ConnectionXmlConfigFileWriter() throws IOException { + file = new File(fileLocation); + conf = new Configuration(false); + try { + if (file.exists()) { + file.delete(); + } + file.createNewFile(); + writer = new PrintWriter(file.getAbsolutePath(), "UTF-8"); + } finally { + file.deleteOnExit(); + } + } + + protected void writeProperty(String key, String value) { + conf.set(key, value); + } + + protected String path() { + return file.getAbsolutePath(); + } + + protected void close() throws IOException { + try { + conf.writeXml(writer); + } finally { + writer.close(); + } + } + } + + @BeforeClass + public static void beforeTest() throws Exception { + Class.forName(MiniHS2.getJdbcDriverName()); + } + + @Before + public void before() throws Exception { + DriverManager.setLoginTimeout(0); + if (!System.getProperty("test.data.files", "").isEmpty()) { + dataFileDir = System.getProperty("test.data.files"); + } + dataFileDir = dataFileDir.replace('\\', '/').replace("c:", ""); + hiveConf = new HiveConf(); + miniHS2 = getNewMiniHS2(); + confOverlay = new HashMap(); + confOverlay.put(ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); + confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, "binary"); + } + + protected MiniHS2 getNewMiniHS2() throws Exception { + return new MiniHS2(hiveConf); + } + + @After + public void tearDown() throws Exception { + if (miniHS2 != null && miniHS2.isStarted()) { + miniHS2.stop(); + } + miniHS2 = null; + System.clearProperty(JAVA_TRUST_STORE_PROP); + System.clearProperty(JAVA_TRUST_STORE_PASS_PROP); + } + + protected void createTable() throws HiveSQLException { + CLIServiceClient serviceClient = miniHS2.getServiceClient(); + SessionHandle sessHandle = serviceClient.openSession("foo", "bar"); + serviceClient.executeStatement(sessHandle, "DROP TABLE IF EXISTS " + tableName, confOverlay); + serviceClient.executeStatement(sessHandle, "CREATE TABLE " + tableName + " (id INT)", + confOverlay); + OperationHandle opHandle = + serviceClient.executeStatement(sessHandle, "SHOW TABLES", confOverlay); + RowSet rowSet = serviceClient.fetchResults(opHandle); + assertFalse(rowSet.numRows() == 0); + } + + protected String testBeeLineConnection(String path, String[] beelineArgs, + String expectedOutput) throws IOException { + TestBeeLine beeLine = null; + try { + if(path != null) { + List testLocations = new ArrayList<>(); + testLocations.add(path); + beeLine = new TestBeeLine(testLocations); + } else { + beeLine = new TestBeeLine(); + } + beeLine.begin(beelineArgs, null); + String output = beeLine.getOutput(); + System.out.println(output); + Assert.assertNotNull(output); + Assert.assertTrue("Output " + output + " does not contain " + expectedOutput, + output.toLowerCase().contains(expectedOutput.toLowerCase())); + return output; + } finally { + if (beeLine != null) { + beeLine.close(); + } + } + } +} diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithUserHs2ConnectionFile.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithUserHs2ConnectionFile.java new file mode 100644 index 0000000000000000000000000000000000000000..0f8618631dbcc6f0d97dbc1bb2b155ecff18b9d4 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithUserHs2ConnectionFile.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hive.beeline.hs2connection; + +import java.io.File; +import java.net.URI; + +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.beeline.hs2connection.HS2ConnectionFileParser; +import org.junit.Test; + +public class TestBeelineWithUserHs2ConnectionFile extends TestBeelineWithHS2ConnectionFile { + + @Test + public void testBeelineConnectionHttp() throws Exception { + setupHttpHs2(); + String path = createHttpHs2ConnectionFile(); + testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); + } + + private void setupHttpHs2() throws Exception { + confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_HTTP_MODE); + confOverlay.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, HS2_HTTP_ENDPOINT); + confOverlay.put(ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true"); + miniHS2.start(confOverlay); + createTable(); + } + + private String createHttpHs2ConnectionFile() throws Exception { + Hs2ConnectionXmlConfigFileWriter writer = new Hs2ConnectionXmlConfigFileWriter(); + String baseJdbcURL = miniHS2.getBaseJdbcURL(); + + URI uri = new URI(baseJdbcURL.substring(5)); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "hosts", + uri.getHost() + ":" + uri.getPort()); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "user", + System.getProperty("user.name")); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "password", + "foo"); + writer.writeProperty( + HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "transportMode", + HS2_HTTP_MODE); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "httpPath", + HS2_HTTP_ENDPOINT); + writer.close(); + return writer.path(); + } + + @Test + public void testBeelineConnectionNoAuth() throws Exception { + setupNoAuthConfHS2(); + String path = createNoAuthHs2ConnectionFile(); + testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); + } + + private void setupNoAuthConfHS2() throws Exception { + // use default configuration for no-auth mode + miniHS2.start(confOverlay); + createTable(); + } + + private String createNoAuthHs2ConnectionFile() throws Exception { + Hs2ConnectionXmlConfigFileWriter writer = new Hs2ConnectionXmlConfigFileWriter(); + String baseJdbcURL = miniHS2.getBaseJdbcURL(); + URI uri = new URI(baseJdbcURL.substring(5)); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "hosts", + uri.getHost() + ":" + uri.getPort()); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "user", + System.getProperty("user.name")); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "password", + "foo"); + writer.close(); + return writer.path(); + } + + @Test + public void testBeelineConnectionSSL() throws Exception { + setupSslHs2(); + String path = createSSLHs2ConnectionFile(); + testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); + } + + private String createSSLHs2ConnectionFile() throws Exception { + Hs2ConnectionXmlConfigFileWriter writer = new Hs2ConnectionXmlConfigFileWriter(); + String baseJdbcURL = miniHS2.getBaseJdbcURL(); + URI uri = new URI(baseJdbcURL.substring(5)); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "hosts", + uri.getHost() + ":" + uri.getPort()); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "user", + System.getProperty("user.name")); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "password", + "foo"); + writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "ssl", + "true"); + writer.writeProperty( + HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "trustStorePassword", + KEY_STORE_TRUST_STORE_PASSWORD); + writer.writeProperty( + HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "sslTrustStore", + dataFileDir + File.separator + TRUST_STORE_NAME); + writer.close(); + return writer.path(); + } + + private void setupSslHs2() throws Exception { + confOverlay.put(ConfVars.HIVE_SERVER2_USE_SSL.varname, "true"); + confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname, + dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME); + confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname, + KEY_STORE_TRUST_STORE_PASSWORD); + miniHS2.start(confOverlay); + createTable(); + } +}