diff --git a/beeline/src/java/org/apache/hive/beeline/Commands.java b/beeline/src/java/org/apache/hive/beeline/Commands.java index 99ee82c..eec2a90 100644 --- a/beeline/src/java/org/apache/hive/beeline/Commands.java +++ b/beeline/src/java/org/apache/hive/beeline/Commands.java @@ -330,7 +330,8 @@ public boolean reconnect(String line) { return beeLine.error(beeLine.loc("no-current-connection")); } } - beeLine.info(beeLine.loc("reconnecting", beeLine.getDatabaseConnection().getUrl())); + beeLine.info(beeLine.loc("reconnecting", + Utils.anonymizeUriString(beeLine.getDatabaseConnection().getUrl()))); try { beeLine.getDatabaseConnection().reconnect(); } catch (Exception e) { @@ -1606,7 +1607,7 @@ public boolean connect(Properties props) throws IOException { } } - beeLine.info("Connecting to " + url); + beeLine.info("Connecting to " + Utils.anonymizeUriString(url)); if (Utils.parsePropertyFromUrl(url, JdbcConnectionParams.AUTH_PRINCIPAL) == null) { String urlForPrompt = url.substring(0, url.contains(";") ? url.indexOf(';') : url.length()); if (username == null) { diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java index 4a82aa5..059a9fa 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java @@ -34,9 +34,7 @@ import org.apache.hive.service.cli.operation.ClassicTableTypeMapping.ClassicTableTypes; import org.apache.hive.service.cli.operation.HiveTableTypeMapping; import org.apache.hive.service.cli.operation.TableTypeMappingFactory.TableTypeMappings; -import org.junit.After; import org.junit.AfterClass; -import org.junit.Before; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java index 1695c5d..dbce6c5 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java +++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java @@ -136,11 +136,7 @@ public HiveConnection(String uri, Properties info) throws SQLException { setupLoginTimeout(); - try { - connParams = Utils.parseURL(uri, info); - } catch (ZooKeeperHiveClientException e) { - throw new SQLException(e); - } + connParams = Utils.parseURL(uri, info); jdbcUriString = connParams.getJdbcUriString(); // JDBC URL: jdbc:hive2://:/dbName;sess_var_list?hive_conf_list#hive_var_list // each list: =;= and so on @@ -175,7 +171,6 @@ public HiveConnection(String uri, Properties info) throws SQLException { EmbeddedThriftBinaryCLIService embeddedClient = new EmbeddedThriftBinaryCLIService(); embeddedClient.init(null); client = embeddedClient; - // open client session openSession(); executeInitSql(); @@ -186,7 +181,8 @@ public HiveConnection(String uri, Properties info) throws SQLException { if (StringUtils.isNotBlank(strRetries)) { maxRetries = Integer.parseInt(strRetries); } - } catch(NumberFormatException e) { // Ignore the exception + } catch(NumberFormatException e) { + // Ignore the exception } for (int numRetries = 0;;) { @@ -198,16 +194,18 @@ public HiveConnection(String uri, Properties info) throws SQLException { // open client session openSession(); executeInitSql(); - break; } catch (Exception e) { - LOG.warn("Failed to connect to " + connParams.getHost() + ":" + connParams.getPort()); + LOG.warn("Failed to connect to " + connParams.getHost() + ":" + connParams.getPort(), e); String errMsg = null; - String warnMsg = "Could not open client transport with JDBC Uri: " + jdbcUriString + ": "; + String warnMsg = + "Could not open client transport with JDBC Uri: " + + Utils.anonymizeUriString(jdbcUriString) + ": "; if (isZkDynamicDiscoveryMode()) { errMsg = "Could not open client transport for any of the Server URI's in ZooKeeper: "; - // Try next available server in zookeeper, or retry all the servers again if retry is enabled - while(!Utils.updateConnParamsFromZooKeeper(connParams) && ++numRetries < maxRetries) { + // Try next available server in zookeeper, or retry all the servers again if retry is + // enabled + while (!Utils.updateConnParamsFromZooKeeper(connParams) && ++numRetries < maxRetries) { connParams.getRejectedHostZnodePaths().clear(); } // Update with new values @@ -452,7 +450,11 @@ public long getRetryInterval() { // Pick trust store config from the given path sslTrustStore = KeyStore.getInstance(JdbcConnectionParams.SSL_TRUST_STORE_TYPE); try (FileInputStream fis = new FileInputStream(sslTrustStorePath)) { - sslTrustStore.load(fis, sslTrustStorePassword.toCharArray()); + if (sslTrustStorePassword != null) { + sslTrustStore.load(fis, sslTrustStorePassword.toCharArray()); + } else { + sslTrustStore.load(fis, null); + } } sslContext = SSLContexts.custom().loadTrustMaterial(sslTrustStore, null).build(); socketFactory = @@ -464,7 +466,8 @@ public long getRetryInterval() { httpClientBuilder.setConnectionManager(new BasicHttpClientConnectionManager(registry)); } catch (Exception e) { String msg = - "Could not create an https connection to " + jdbcUriString + ". " + e.getMessage(); + "Could not create an https connection to " + Utils.anonymizeUriString(jdbcUriString) + + ". " + e.getMessage(); throw new SQLException(msg, " 08S01", e); } } @@ -561,7 +564,7 @@ private TTransport createBinaryTransport() throws SQLException, TTransportExcept } } catch (SaslException e) { throw new SQLException("Could not create secure connection to " - + jdbcUriString + ": " + e.getMessage(), " 08S01", e); + + Utils.anonymizeUriString(jdbcUriString) + ": " + e.getMessage(), " 08S01", e); } return transport; } @@ -678,7 +681,7 @@ private void openSession() throws SQLException { } catch (TException e) { LOG.error("Error opening session", e); throw new SQLException("Could not establish connection to " - + jdbcUriString + ": " + e.getMessage(), " 08S01", e); + + Utils.anonymizeUriString(jdbcUriString) + ": " + e.getMessage(), " 08S01", e); } isClosed = false; } diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveDriver.java b/jdbc/src/java/org/apache/hive/jdbc/HiveDriver.java index a349f8b..cf5e5a2 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/HiveDriver.java +++ b/jdbc/src/java/org/apache/hive/jdbc/HiveDriver.java @@ -231,17 +231,11 @@ public boolean jdbcCompliant() { private Properties parseURLforPropertyInfo(String url, Properties defaults) throws SQLException { Properties urlProps = (defaults != null) ? new Properties(defaults) : new Properties(); - if (url == null || !url.startsWith(Utils.URL_PREFIX)) { throw new SQLException("Invalid connection url: " + url); } - JdbcConnectionParams params = null; - try { - params = Utils.parseURL(url, defaults); - } catch (ZooKeeperHiveClientException e) { - throw new SQLException(e); - } + params = Utils.parseURL(url, defaults); String host = params.getHost(); if (host == null){ host = ""; diff --git a/jdbc/src/java/org/apache/hive/jdbc/Utils.java b/jdbc/src/java/org/apache/hive/jdbc/Utils.java index bfae8b9..30b4e98 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/Utils.java +++ b/jdbc/src/java/org/apache/hive/jdbc/Utils.java @@ -268,7 +268,7 @@ static void verifySuccess(TStatus status, boolean withInfo) throws SQLException } public static JdbcConnectionParams parseURL(String uri) throws JdbcUriParseException, - SQLException, ZooKeeperHiveClientException { + SQLException { return parseURL(uri, new Properties()); } /** @@ -294,12 +294,14 @@ public static JdbcConnectionParams parseURL(String uri) throws JdbcUriParseExcep * @return * @throws SQLException */ - static JdbcConnectionParams parseURL(String uri, Properties info) throws JdbcUriParseException, - SQLException, ZooKeeperHiveClientException { + static JdbcConnectionParams parseURL(String uri, Properties connProperties) + throws JdbcUriParseException { + String anonymizedUri = anonymizeUriString(uri); JdbcConnectionParams connParams = new JdbcConnectionParams(); - + LOG.info("Parsing JDBC uri: " + anonymizedUri); if (!uri.startsWith(URL_PREFIX)) { - throw new JdbcUriParseException("Bad URL format: Missing prefix " + URL_PREFIX); + throw new JdbcUriParseException("Bad URL format: Missing prefix " + URL_PREFIX + + ". Supplied JDBC Uri: " + anonymizedUri); } // For URLs with no other configuration @@ -322,7 +324,6 @@ static JdbcConnectionParams parseURL(String uri, Properties info) throws JdbcUri // jdbc:hive2:///dbName;sess_var_list?hive_conf_list#hive_var_list connParams.setEmbeddedMode(true); } else { - LOG.info("Supplied authorities: " + suppliedAuthorities); String[] authorityList = suppliedAuthorities.split(","); connParams.setSuppliedAuthorityList(authorityList); uri = uri.replace(suppliedAuthorities, dummyAuthorityString); @@ -346,16 +347,9 @@ static JdbcConnectionParams parseURL(String uri, Properties info) throws JdbcUri } else { // we have dbname followed by session parameters dbName = sessVars.substring(0, sessVars.indexOf(';')); + // parse session vars sessVars = sessVars.substring(sessVars.indexOf(';') + 1); - if (sessVars != null) { - Matcher sessMatcher = pattern.matcher(sessVars); - while (sessMatcher.find()) { - if (connParams.getSessionVars().put(sessMatcher.group(1), sessMatcher.group(2)) != null) { - throw new JdbcUriParseException("Bad URL format: Multiple values for property " - + sessMatcher.group(1)); - } - } - } + setParams(anonymizedUri, pattern, sessVars, connParams.getSessionVars()); } if (!dbName.isEmpty()) { connParams.setDbName(dbName); @@ -364,53 +358,113 @@ static JdbcConnectionParams parseURL(String uri, Properties info) throws JdbcUri // parse hive conf settings String confStr = jdbcURI.getQuery(); - if (confStr != null) { - Matcher confMatcher = pattern.matcher(confStr); - while (confMatcher.find()) { - connParams.getHiveConfs().put(confMatcher.group(1), confMatcher.group(2)); - } - } + setParams(anonymizedUri, pattern, confStr, connParams.getHiveConfs()); // parse hive var settings String varStr = jdbcURI.getFragment(); - if (varStr != null) { - Matcher varMatcher = pattern.matcher(varStr); - while (varMatcher.find()) { - connParams.getHiveVars().put(varMatcher.group(1), varMatcher.group(2)); + setParams(anonymizedUri, pattern, varStr, connParams.getHiveVars()); + + // Override connection params with those supplied in connection properties + applyConnectionProps(connParams, connProperties); + + handleDeprecations(connParams); + + // Extract host, port + if (connParams.isEmbeddedMode()) { + // In case of embedded mode we were supplied with an empty authority. + // So we never substituted the authority with a dummy one. + connParams.setHost(jdbcURI.getHost()); + connParams.setPort(jdbcURI.getPort()); + } else { + // Configure host, port and params from ZooKeeper if used, + // and substitute the dummy authority with a resolved one + try { + configureConnParams(connParams); + } catch (JdbcUriParseException | ZooKeeperHiveClientException e) { + throw new JdbcUriParseException("Error parsing JDBC Uri: " + anonymizedUri, e); } + // We check for invalid host, port while configuring connParams with configureConnParams() + String authorityStr = connParams.getHost() + ":" + connParams.getPort(); + LOG.info("Resolved authority: " + authorityStr); + uri = uri.replace(dummyAuthorityString, authorityStr); + connParams.setJdbcUriString(uri); } - - // Apply configs supplied in the JDBC connection properties object - for (Map.Entry kv : info.entrySet()) { + + return connParams; + } + + // Remove sensitive information (if present) from the connection string that gets logged + public static String anonymizeUriString(String uriString) { + String anonymizedUriString = new String(uriString); + // Remove username value + anonymizedUriString = + anonymizedUriString.replaceAll(";(?i)" + JdbcConnectionParams.AUTH_USER + "=[^;?#]*", ";" + + JdbcConnectionParams.AUTH_USER + "=xxxxx"); + // Remove password value + anonymizedUriString = + anonymizedUriString.replaceAll(";(?i)" + JdbcConnectionParams.AUTH_PASSWD + "=[^;?#]*", ";" + + JdbcConnectionParams.AUTH_PASSWD + "=xxxxx"); + // Remove ssl keystore password value + anonymizedUriString = + anonymizedUriString.replaceAll(";(?i)" + JdbcConnectionParams.SSL_KEY_STORE_PASSWORD + + "=[^;?#]*", ";" + JdbcConnectionParams.SSL_KEY_STORE_PASSWORD + "=xxxxx"); + // Remove ssl truststore password value + anonymizedUriString = + anonymizedUriString.replaceAll(";(?i)" + JdbcConnectionParams.SSL_TRUST_STORE_PASSWORD + + "=[^;?#]*", ";" + JdbcConnectionParams.SSL_TRUST_STORE_PASSWORD + "=xxxxx"); + // Remove http header username value + anonymizedUriString = + anonymizedUriString.replaceAll(";(?i)" + JdbcConnectionParams.HTTP_HEADER_PREFIX + + "USERNAME=[^;?#]*", ";" + JdbcConnectionParams.HTTP_HEADER_PREFIX + "USERNAME=xxxxx"); + // Remove http header password value + anonymizedUriString = + anonymizedUriString.replaceAll(";(?i)" + JdbcConnectionParams.HTTP_HEADER_PREFIX + + "PASSWORD=[^;?#]*", ";" + JdbcConnectionParams.HTTP_HEADER_PREFIX + "PASSWORD=xxxxx"); + // Remove hive.conf.hidden.list value + anonymizedUriString = + anonymizedUriString.replaceAll("hive.conf.hidden.list=[^;?#]*", + "hive.conf.hidden.list=xxxxx"); + return anonymizedUriString; + } + + // Apply configs supplied in the JDBC connection properties object + private static void applyConnectionProps(JdbcConnectionParams connParams, + Properties connProperties) { + for (Map.Entry kv : connProperties.entrySet()) { if ((kv.getKey() instanceof String)) { String key = (String) kv.getKey(); if (key.startsWith(JdbcConnectionParams.HIVE_VAR_PREFIX)) { connParams.getHiveVars().put( - key.substring(JdbcConnectionParams.HIVE_VAR_PREFIX.length()), info.getProperty(key)); + key.substring(JdbcConnectionParams.HIVE_VAR_PREFIX.length()), + connProperties.getProperty(key)); } else if (key.startsWith(JdbcConnectionParams.HIVE_CONF_PREFIX)) { connParams.getHiveConfs().put( - key.substring(JdbcConnectionParams.HIVE_CONF_PREFIX.length()), info.getProperty(key)); + key.substring(JdbcConnectionParams.HIVE_CONF_PREFIX.length()), + connProperties.getProperty(key)); } } } + // Extract user/password from JDBC connection properties if its not supplied // in the connection URL if (!connParams.getSessionVars().containsKey(JdbcConnectionParams.AUTH_USER)) { - if (info.containsKey(JdbcConnectionParams.AUTH_USER)) { - connParams.getSessionVars().put(JdbcConnectionParams.AUTH_USER, - info.getProperty(JdbcConnectionParams.AUTH_USER)); - } - if (info.containsKey(JdbcConnectionParams.AUTH_PASSWD)) { - connParams.getSessionVars().put(JdbcConnectionParams.AUTH_PASSWD, - info.getProperty(JdbcConnectionParams.AUTH_PASSWD)); - } + if (connProperties.containsKey(JdbcConnectionParams.AUTH_USER)) { + connParams.getSessionVars().put(JdbcConnectionParams.AUTH_USER, + connProperties.getProperty(JdbcConnectionParams.AUTH_USER)); + } + if (connProperties.containsKey(JdbcConnectionParams.AUTH_PASSWD)) { + connParams.getSessionVars().put(JdbcConnectionParams.AUTH_PASSWD, + connProperties.getProperty(JdbcConnectionParams.AUTH_PASSWD)); + } } - if (info.containsKey(JdbcConnectionParams.AUTH_TYPE)) { + if (connProperties.containsKey(JdbcConnectionParams.AUTH_TYPE)) { connParams.getSessionVars().put(JdbcConnectionParams.AUTH_TYPE, - info.getProperty(JdbcConnectionParams.AUTH_TYPE)); + connProperties.getProperty(JdbcConnectionParams.AUTH_TYPE)); } + } + private static void handleDeprecations(JdbcConnectionParams connParams) { // Handle all deprecations here: String newUsage; String usageUrlBase = "jdbc:hive2://:/dbName;"; @@ -429,23 +483,21 @@ static JdbcConnectionParams parseURL(String uri, Properties info) throws JdbcUri newUsage = usageUrlBase + JdbcConnectionParams.HTTP_PATH + "="; handleParamDeprecation(connParams.getHiveConfs(), connParams.getSessionVars(), JdbcConnectionParams.HTTP_PATH_DEPRECATED, JdbcConnectionParams.HTTP_PATH, newUsage); - // Extract host, port - if (connParams.isEmbeddedMode()) { - // In case of embedded mode we were supplied with an empty authority. - // So we never substituted the authority with a dummy one. - connParams.setHost(jdbcURI.getHost()); - connParams.setPort(jdbcURI.getPort()); - } else { - // Configure host, port and params from ZooKeeper if used, - // and substitute the dummy authority with a resolved one - configureConnParams(connParams); - // We check for invalid host, port while configuring connParams with configureConnParams() - String authorityStr = connParams.getHost() + ":" + connParams.getPort(); - LOG.info("Resolved authority: " + authorityStr); - uri = uri.replace(dummyAuthorityString, authorityStr); - connParams.setJdbcUriString(uri); + } + + private static void setParams(String anonymizedUri, Pattern pattern, String paramStr, + Map paramGroup) throws JdbcUriParseException { + if (paramStr != null) { + Matcher confMatcher = pattern.matcher(paramStr); + while (confMatcher.find()) { + // If there a property is specified multiple times in the URI, we can't decide which + // key-value pair to pick. Throw error. + if (paramGroup.put(confMatcher.group(1), confMatcher.group(2)) != null) { + throw new JdbcUriParseException("Bad URL format: Multiple values for property: " + + confMatcher.group(1) + ". Supplied JDBC Uri: " + anonymizedUri); + } + } } - return connParams; } /** diff --git a/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java b/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java index 8d6003a..206834b 100644 --- a/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java +++ b/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java @@ -70,6 +70,7 @@ static void configureConnParams(JdbcConnectionParams connParams) } // Now pick a server node randomly serverNode = serverHosts.get(randomizer.nextInt(serverHosts.size())); + LOG.info("Picked server: " + serverNode + " from ZooKeeper. Trying to configure it now."); connParams.setCurrentHostZnodePath(serverNode); // Read data from the znode for this server node // This data could be either config string (new releases) or server end @@ -93,7 +94,7 @@ static void configureConnParams(JdbcConnectionParams connParams) applyConfs(dataStr, connParams); } } catch (Exception e) { - throw new ZooKeeperHiveClientException("Unable to read HiveServer2 configs from ZooKeeper", e); + throw new ZooKeeperHiveClientException(e); } finally { // Close the client connection with ZooKeeper if (zooKeeperClient != null) { diff --git a/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver.java b/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver.java index 162e42f..079fe1d 100644 --- a/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver.java +++ b/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver.java @@ -34,6 +34,8 @@ import java.util.Collection; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; @RunWith(Parameterized.class) public class TestJdbcDriver { @@ -95,4 +97,35 @@ public void testParseInitFile() throws IOException { } } } + + @Test + public void testJdbcUrlAnonymization() { + String username = "user1"; + String password = "pwd"; + String trustStorePassword = "tpwd"; + String keyStorePassword = "kpwd"; + String hiddenConf = "hidden"; + String transportMode = "http"; + String httpPath = "cliservice"; + String operationLogEnabled = "log.enabled"; + String jdbcUrl = + "jdbc:hive2://localhost:10000/;user=" + username + ";password=" + password + + ";transportMode=" + transportMode + ";httpPath=" + httpPath + ";trustStorePassword=" + + trustStorePassword + ";keyStorePassword=" + keyStorePassword + + ";http.header.USERNAME=" + username + ";http.header.PASSWORD=" + password + + "?hive.conf.hidden.list=" + hiddenConf + ";hive.server2.logging.operation.enabled=" + + operationLogEnabled; + String anonymizedUrl = Utils.anonymizeUriString(jdbcUrl); + assertFalse(anonymizedUrl.contains(username)); + assertFalse(anonymizedUrl.contains(password)); + assertFalse(anonymizedUrl.contains(trustStorePassword)); + assertFalse(anonymizedUrl.contains(keyStorePassword)); + assertFalse(anonymizedUrl.contains(hiddenConf)); + assertTrue(anonymizedUrl.contains("transportMode")); + assertTrue(anonymizedUrl.contains(transportMode)); + assertTrue(anonymizedUrl.contains("httpPath")); + assertTrue(anonymizedUrl.contains(httpPath)); + assertTrue(anonymizedUrl.contains("hive.server2.logging.operation.enabled")); + assertTrue(anonymizedUrl.contains(operationLogEnabled)); + } }