commit c8abe9eea073702be1d4553e304d14fab132f799 Author: Janaki Lahorani Date: Mon Apr 16 14:42:04 2018 -0700 HIVE-19228: Remove dependency on commons-httpclient. Change-Id: Iac5bf8bb1d56c29d00ab108f2e727d572485aa09 diff --git itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestActivePassiveHA.java itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestActivePassiveHA.java index c55271f038e36bffe523acfc1b092bb892af6c13..4055f13fd083950c94013c41a9c23bb2f88a4d43 100644 --- itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestActivePassiveHA.java +++ itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestActivePassiveHA.java @@ -36,11 +36,6 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpMethodBase; -import org.apache.commons.httpclient.methods.DeleteMethod; -import org.apache.commons.httpclient.methods.GetMethod; -import org.apache.commons.httpclient.methods.OptionsMethod; import org.apache.curator.test.TestingServer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; @@ -54,9 +49,22 @@ import org.apache.hive.service.server.HiveServer2Instance; import org.apache.hive.service.server.TestHS2HttpServerPam; import org.apache.hive.service.servlet.HS2Peers; +import org.apache.http.Header; import org.apache.http.HttpException; import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpOptions; +import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.StatusLine; +import org.apache.http.util.EntityUtils; import org.codehaus.jackson.map.ObjectMapper; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.util.B64Code; +import org.eclipse.jetty.util.StringUtil; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -406,7 +414,7 @@ public void testManualFailoverUnauthorized() throws Exception { assertEquals("true", sendGet(url1, true)); // trigger failover on miniHS2_1 without authorization header - assertEquals("Unauthorized", sendDelete(url1, false)); + assertTrue(sendDelete(url1, false).contains("Unauthorized")); assertTrue(sendDelete(url1, true).contains("Failover successful!")); assertEquals(true, miniHS2_1.getNotLeaderTestFuture().get()); assertEquals(false, miniHS2_1.isLeader()); @@ -541,56 +549,79 @@ private String sendGet(String url) throws Exception { } private String sendGet(String url, boolean enableAuth) throws Exception { - return sendAuthMethod(new GetMethod(url), enableAuth, false); + return sendAuthMethod(new HttpGet(url), enableAuth, false); } private String sendGet(String url, boolean enableAuth, boolean enableCORS) throws Exception { - return sendAuthMethod(new GetMethod(url), enableAuth, enableCORS); + return sendAuthMethod(new HttpGet(url), enableAuth, enableCORS); } private String sendDelete(String url, boolean enableAuth) throws Exception { - return sendAuthMethod(new DeleteMethod(url), enableAuth, false); + return sendAuthMethod(new HttpDelete(url), enableAuth, false); } private String sendDelete(String url, boolean enableAuth, boolean enableCORS) throws Exception { - return sendAuthMethod(new DeleteMethod(url), enableAuth, enableCORS); + return sendAuthMethod(new HttpDelete(url), enableAuth, enableCORS); } - private String sendAuthMethod(HttpMethodBase method, boolean enableAuth, boolean enableCORS) throws Exception { - HttpClient client = new HttpClient(); - try { - if (enableAuth) { - setupAuthHeaders(method); - } + private String sendAuthMethod(HttpRequestBase method, boolean enableAuth, boolean enableCORS) throws Exception { + CloseableHttpResponse httpResponse = null; + + try ( + CloseableHttpClient client = HttpClients.createDefault(); + ) { + // CORS check if (enableCORS) { String origin = "http://example.com"; - OptionsMethod optionsMethod = new OptionsMethod(method.getURI().toString()); - optionsMethod.addRequestHeader("Origin", origin); - setupAuthHeaders(optionsMethod); - int statusCode = client.executeMethod(optionsMethod); - if (statusCode == 200) { - assertNotNull(optionsMethod.getResponseHeader("Access-Control-Allow-Origin")); - assertEquals(origin, optionsMethod.getResponseHeader("Access-Control-Allow-Origin").getValue()); + + HttpOptions optionsMethod = new HttpOptions(method.getURI().toString()); + + optionsMethod.addHeader("Origin", origin); + + if (enableAuth) { + setupAuthHeaders(optionsMethod); + } + + httpResponse = client.execute(optionsMethod); + + if (httpResponse != null) { + StatusLine statusLine = httpResponse.getStatusLine(); + if (statusLine != null) { + String response = httpResponse.getStatusLine().getReasonPhrase(); + int statusCode = httpResponse.getStatusLine().getStatusCode(); + + if (statusCode == 200) { + Header originHeader = httpResponse.getFirstHeader("Access-Control-Allow-Origin"); + assertNotNull(originHeader); + assertEquals(origin, originHeader.getValue()); + } else { + fail("CORS returned: " + statusCode + " Error: " + response); + } + } else { + fail("Status line is null"); + } } else { - fail("CORS returned: " + statusCode + " Error: " + optionsMethod.getStatusLine().getReasonPhrase()); + fail("No http Response"); } } - int statusCode = client.executeMethod(method); - if (statusCode == 200) { - return method.getResponseBodyAsString(); - } else { - return method.getStatusLine().getReasonPhrase(); + + if (enableAuth) { + setupAuthHeaders(method); } + + httpResponse = client.execute(method); + + return EntityUtils.toString(httpResponse.getEntity()); } finally { - method.releaseConnection(); + httpResponse.close(); } } - private void setupAuthHeaders(final HttpMethodBase method) { - String userPass = ADMIN_USER + ":" + ADMIN_PASSWORD; - method.addRequestHeader(HttpHeaders.AUTHORIZATION, - "Basic " + new String(Base64.getEncoder().encode(userPass.getBytes()))); + private void setupAuthHeaders(final HttpRequestBase method) { + String authB64Code = + B64Code.encode(ADMIN_USER + ":" + ADMIN_PASSWORD, StringUtil.__ISO_8859_1); + method.setHeader(HttpHeader.AUTHORIZATION.asString(), "Basic " + authB64Code); } private Map getConfOverlay(final String instanceId) { diff --git pom.xml pom.xml index e11339982d842addee00f3402da994de49136132..ce3da37d2b4a33819fe62c32589c417ad5dd17fd 100644 --- pom.xml +++ pom.xml @@ -133,7 +133,6 @@ 3.2.2 1.9 1.1 - 3.0.1 2.4 2.6 3.2 @@ -358,11 +357,6 @@ commons-collections ${commons-collections.version} - - commons-httpclient - commons-httpclient - ${commons-httpclient.version} - commons-io commons-io diff --git ql/pom.xml ql/pom.xml index 867a38aa68c80dab9a23b6a3d8b392bcb40a0cf1..fedb5f1f80e8aab570a9c6a63272f9327199eb02 100644 --- ql/pom.xml +++ ql/pom.xml @@ -108,21 +108,6 @@ commons-codec ${commons-codec.version} - - commons-httpclient - commons-httpclient - ${commons-httpclient.version} - - - org.slf4j - slf4j-log4j12 - - - commmons-logging - commons-logging - - - commons-io commons-io diff --git ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index 2b88ea651bde32ed8b755a0c3789ed43e6611aa1..3ab1e22f3f6889e4243d643900ad5298eca67bfa 100644 --- ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.ql.parse; +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.net.URLCodec; import org.apache.hadoop.hive.conf.HiveConf.StrictChecks; import java.io.IOException; import java.io.Serializable; @@ -31,7 +33,6 @@ import java.util.HashSet; import org.antlr.runtime.tree.Tree; -import org.apache.commons.httpclient.util.URIUtil; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -107,8 +108,8 @@ public boolean accept(Path p) { return (srcs); } - private URI initializeFromURI(String fromPath, boolean isLocal) throws IOException, - URISyntaxException { + private URI initializeFromURI(String fromPath, boolean isLocal) + throws IOException, URISyntaxException, SemanticException { URI fromURI = new Path(fromPath).toUri(); String fromScheme = fromURI.getScheme(); @@ -119,8 +120,13 @@ private URI initializeFromURI(String fromPath, boolean isLocal) throws IOExcepti // directory if (!path.startsWith("/")) { if (isLocal) { - path = URIUtil.decode( - new Path(System.getProperty("user.dir"), fromPath).toUri().toString()); + try { + path = new String(URLCodec.decodeUrl( + new Path(System.getProperty("user.dir"), fromPath).toUri().toString() + .getBytes("US-ASCII")), "US-ASCII"); + } catch (DecoderException de) { + throw new SemanticException("URL Decode failed", de); + } } else { path = new Path(new Path("/user/" + System.getProperty("user.name")), path).toString(); diff --git service/src/java/org/apache/hive/service/server/HiveServer2.java service/src/java/org/apache/hive/service/server/HiveServer2.java index cf3edbf47e4802e1e73c65a2ef52e45ca44c9fed..661beb54e45f145515044e9f85db1047eda200bb 100644 --- service/src/java/org/apache/hive/service/server/HiveServer2.java +++ service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -45,9 +45,6 @@ import org.apache.commons.cli.OptionBuilder; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import org.apache.commons.httpclient.HttpClient; -import org.apache.commons.httpclient.HttpMethodBase; -import org.apache.commons.httpclient.methods.DeleteMethod; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.concurrent.BasicThreadFactory; import org.apache.curator.framework.CuratorFramework; @@ -109,7 +106,12 @@ import org.apache.hive.service.servlet.HS2LeadershipStatus; import org.apache.hive.service.servlet.HS2Peers; import org.apache.hive.service.servlet.QueryProfileServlet; -import org.apache.http.HttpHeaders; +import org.apache.http.StatusLine; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.util.Strings; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -1377,22 +1379,38 @@ public void execute() { // invoke DELETE /leader endpoint for failover String webEndpoint = "http://" + targetInstance.getHost() + ":" + webPort + "/leader"; - HttpClient client = new HttpClient(); - HttpMethodBase method = new DeleteMethod(webEndpoint); - try { - int statusCode = client.executeMethod(method); - if (statusCode == 200) { - System.out.println(method.getResponseBodyAsString()); - } else { - String response = method.getStatusLine().getReasonPhrase(); - LOG.error("Unable to failover HiveServer2 instance: " + workerIdentity + ". status code: " + - statusCode + "error: " + response); - System.err.println("Unable to failover HiveServer2 instance: " + workerIdentity + ". status code: " + - statusCode + " error: " + response); + HttpDelete httpDelete = new HttpDelete(webEndpoint); + CloseableHttpResponse httpResponse = null; + try ( + CloseableHttpClient client = HttpClients.createDefault(); + ) { + int statusCode = -1; + String response = "Response unavailable"; + httpResponse = client.execute(httpDelete); + if (httpResponse != null) { + StatusLine statusLine = httpResponse.getStatusLine(); + if (statusLine != null) { + response = httpResponse.getStatusLine().getReasonPhrase(); + statusCode = httpResponse.getStatusLine().getStatusCode(); + + if (statusCode == 200) { + System.out.println(EntityUtils.toString(httpResponse.getEntity())); + } + } + } + + if (statusCode != 200) { + // Failover didn't succeed - log error and exit + LOG.error("Unable to failover HiveServer2 instance: " + workerIdentity + + ". status code: " + statusCode + "error: " + response); + System.err.println("Unable to failover HiveServer2 instance: " + workerIdentity + + ". status code: " + statusCode + " error: " + response); System.exit(-1); } } finally { - method.releaseConnection(); + if (httpResponse != null) { + httpResponse.close(); + } } } catch (IOException e) { LOG.error("Error listing HiveServer2 HA instances from ZooKeeper", e);