commit 1daf81455e8ca66556af0a8f8a1b3b739ca1c2fc Author: Janaki Lahorani Date: Mon Apr 16 14:42:04 2018 -0700 HIVE-19228: Remove dependency on commons-httpclient. Change-Id: Iac5bf8bb1d56c29d00ab108f2e727d572485aa09 diff --git itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestActivePassiveHA.java itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestActivePassiveHA.java index fb846b49a596929df2bb30b3a19b7720001498f7..97632e3e00f0019b04d97d5815bdafb663f29dde 100644 --- itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestActivePassiveHA.java +++ itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestActivePassiveHA.java @@ -27,17 +27,12 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; -import java.util.Base64; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpMethodBase; -import org.apache.commons.httpclient.methods.DeleteMethod; -import org.apache.commons.httpclient.methods.GetMethod; import org.apache.curator.test.TestingServer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -51,8 +46,17 @@ import org.apache.hive.service.server.HiveServer2Instance; import org.apache.hive.service.server.TestHS2HttpServerPam; import org.apache.hive.service.servlet.HS2Peers; -import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; import org.codehaus.jackson.map.ObjectMapper; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.util.B64Code; +import org.eclipse.jetty.util.StringUtil; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -398,7 +402,7 @@ public void testManualFailoverUnauthorized() throws Exception { assertEquals("true", sendGet(url1, true)); // trigger failover on miniHS2_1 without authorization header - assertEquals("Unauthorized", sendDelete(url1, false)); + assertTrue(sendDelete(url1, false).contains("Unauthorized")); assertTrue(sendDelete(url1, true).contains("Failover successful!")); assertEquals(true, miniHS2_1.getNotLeaderTestFuture().get()); assertEquals(false, miniHS2_1.isLeader()); @@ -533,29 +537,28 @@ private String sendGet(String url) throws Exception { } private String sendGet(String url, boolean enableAuth) throws Exception { - return sendAuthMethod(new GetMethod(url), enableAuth); + return sendAuthMethod(new HttpGet(url), enableAuth); } private String sendDelete(String url, boolean enableAuth) throws Exception { - return sendAuthMethod(new DeleteMethod(url), enableAuth); + return sendAuthMethod(new HttpDelete(url), enableAuth); } - private String sendAuthMethod(HttpMethodBase method, boolean enableAuth) throws Exception { - HttpClient client = new HttpClient(); - try { + private String sendAuthMethod(HttpRequestBase method, boolean enableAuth) throws Exception { + CloseableHttpResponse httpResponse = null; + try ( + CloseableHttpClient client = HttpClients.createDefault(); + ) { if (enableAuth) { - String userPass = ADMIN_USER + ":" + ADMIN_PASSWORD; - method.addRequestHeader(HttpHeaders.AUTHORIZATION, - "Basic " + new String(Base64.getEncoder().encode(userPass.getBytes()))); - } - int statusCode = client.executeMethod(method); - if (statusCode == 200) { - return method.getResponseBodyAsString(); - } else { - return method.getStatusLine().getReasonPhrase(); + String authB64Code = + B64Code.encode(ADMIN_USER + ":" + ADMIN_PASSWORD, StringUtil.__ISO_8859_1); + method.setHeader(HttpHeader.AUTHORIZATION.asString(), "Basic " + authB64Code); } + httpResponse = client.execute(method); + + return EntityUtils.toString(httpResponse.getEntity()); } finally { - method.releaseConnection(); + httpResponse.close(); } } diff --git pom.xml pom.xml index e11339982d842addee00f3402da994de49136132..ce3da37d2b4a33819fe62c32589c417ad5dd17fd 100644 --- pom.xml +++ pom.xml @@ -133,7 +133,6 @@ 3.2.2 1.9 1.1 - 3.0.1 2.4 2.6 3.2 @@ -358,11 +357,6 @@ commons-collections ${commons-collections.version} - - commons-httpclient - commons-httpclient - ${commons-httpclient.version} - commons-io commons-io diff --git ql/pom.xml ql/pom.xml index 867a38aa68c80dab9a23b6a3d8b392bcb40a0cf1..fedb5f1f80e8aab570a9c6a63272f9327199eb02 100644 --- ql/pom.xml +++ ql/pom.xml @@ -108,21 +108,6 @@ commons-codec ${commons-codec.version} - - commons-httpclient - commons-httpclient - ${commons-httpclient.version} - - - org.slf4j - slf4j-log4j12 - - - commmons-logging - commons-logging - - - commons-io commons-io diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index 550fe5075ebf92dbe5ff63aa69acd7bde6d8616e..7e1766e066f62c9a5d43915b35f662d21158823b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.parse; +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.net.URLCodec; import org.apache.hadoop.hive.conf.HiveConf.StrictChecks; import java.io.IOException; import java.io.Serializable; @@ -31,7 +33,6 @@ import java.util.HashSet; import org.antlr.runtime.tree.Tree; -import org.apache.commons.httpclient.util.URIUtil; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -106,8 +107,8 @@ public boolean accept(Path p) { return (srcs); } - private URI initializeFromURI(String fromPath, boolean isLocal) throws IOException, - URISyntaxException { + private URI initializeFromURI(String fromPath, boolean isLocal) + throws IOException, URISyntaxException, SemanticException { URI fromURI = new Path(fromPath).toUri(); String fromScheme = fromURI.getScheme(); @@ -118,8 +119,13 @@ private URI initializeFromURI(String fromPath, boolean isLocal) throws IOExcepti // directory if (!path.startsWith("/")) { if (isLocal) { - path = URIUtil.decode( - new Path(System.getProperty("user.dir"), fromPath).toUri().toString()); + try { + path = new String(URLCodec.decodeUrl( + new Path(System.getProperty("user.dir"), fromPath).toUri().toString() + .getBytes("US-ASCII")), "US-ASCII"); + } catch (DecoderException de) { + throw new SemanticException("URL Decode failed", de); + } } else { path = new Path(new Path("/user/" + System.getProperty("user.name")), path).toString(); diff --git service/src/java/org/apache/hive/service/server/HiveServer2.java service/src/java/org/apache/hive/service/server/HiveServer2.java index e373628ae195cd32cb0866432405334a40314023..06ea063bc985ec7755706332136481bf88d4f495 100644 --- service/src/java/org/apache/hive/service/server/HiveServer2.java +++ service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -45,9 +45,6 @@ import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpMethodBase; -import org.apache.commons.httpclient.methods.DeleteMethod; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.apache.curator.framework.CuratorFramework; @@ -106,7 +103,12 @@ import org.apache.hive.service.servlet.HS2LeadershipStatus; import org.apache.hive.service.servlet.HS2Peers; import org.apache.hive.service.servlet.QueryProfileServlet; -import org.apache.http.HttpHeaders; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.util.Strings; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -1314,22 +1316,38 @@ public void execute() { // invoke DELETE /leader endpoint for failover String webEndpoint = "http://" + targetInstance.getHost() + ":" + webPort + "/leader"; - HttpClient client = new HttpClient(); - HttpMethodBase method = new DeleteMethod(webEndpoint); - try { - int statusCode = client.executeMethod(method); - if (statusCode == 200) { - System.out.println(method.getResponseBodyAsString()); - } else { - String response = method.getStatusLine().getReasonPhrase(); - LOG.error("Unable to failover HiveServer2 instance: " + workerIdentity + ". status code: " + - statusCode + "error: " + response); - System.err.println("Unable to failover HiveServer2 instance: " + workerIdentity + ". status code: " + - statusCode + " error: " + response); + HttpDelete httpDelete = new HttpDelete(webEndpoint); + CloseableHttpResponse httpResponse = null; + try ( + CloseableHttpClient client = HttpClients.createDefault(); + ) { + int statusCode = -1; + String response = "Response unavailable"; + httpResponse = client.execute(httpDelete); + if (httpResponse != null) { + StatusLine statusLine = httpResponse.getStatusLine(); + if (statusLine != null) { + response = httpResponse.getStatusLine().getReasonPhrase(); + statusCode = httpResponse.getStatusLine().getStatusCode(); + + if (statusCode == 200) { + System.out.println(EntityUtils.toString(httpResponse.getEntity())); + } + } + } + + if (statusCode != 200) { + // Failover didn't succeed - log error and exit + LOG.error("Unable to failover HiveServer2 instance: " + workerIdentity + + ". status code: " + statusCode + "error: " + response); + System.err.println("Unable to failover HiveServer2 instance: " + workerIdentity + + ". status code: " + statusCode + " error: " + response); System.exit(-1); } } finally { - method.releaseConnection(); + if (httpResponse != null) { + httpResponse.close(); + } } } catch (IOException e) { LOG.error("Error listing HiveServer2 HA instances from ZooKeeper", e);