diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftBinaryCLIService.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftBinaryCLIService.java deleted file mode 100644 index de31699..0000000 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftBinaryCLIService.java +++ /dev/null @@ -1,92 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hive.service.cli.thrift; - -import static org.junit.Assert.assertNotNull; - -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hive.service.auth.HiveAuthFactory.AuthTypes; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; - - -/** - * - * TestThriftBinaryCLIService. - * This tests ThriftCLIService started in binary mode. - * - */ - -public class TestThriftBinaryCLIService extends ThriftCLIServiceTest { - - private static String transportMode = "binary"; - - /** - * @throws java.lang.Exception - */ - @BeforeClass - public static void setUpBeforeClass() throws Exception { - // Set up the base class - ThriftCLIServiceTest.setUpBeforeClass(); - - assertNotNull(port); - assertNotNull(hiveServer2); - assertNotNull(hiveConf); - - hiveConf.setBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); - hiveConf.setVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, host); - hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, port); - hiveConf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, AuthTypes.NONE.toString()); - hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, transportMode); - - startHiveServer2WithConf(hiveConf); - - client = getServiceClientInternal(); - } - - /** - * @throws java.lang.Exception - */ - @AfterClass - public static void tearDownAfterClass() throws Exception { - ThriftCLIServiceTest.tearDownAfterClass(); - } - - /** - * @throws java.lang.Exception - */ - @Override - @Before - public void setUp() throws Exception { - - } - - /** - * @throws java.lang.Exception - */ - @Override - @After - public void tearDown() throws Exception { - - } - - -} \ No newline at end of file diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithBinary.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithBinary.java new file mode 100644 index 0000000..ceaf0e92 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithBinary.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.service.cli.thrift; + +import static org.junit.Assert.assertNotNull; + +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.service.auth.HiveAuthFactory.AuthTypes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + + +/** + * + * TestThriftBinaryCLIService. + * This tests ThriftCLIService started in binary mode. + * + */ + +public class TestThriftCLIServiceWithBinary extends ThriftCLIServiceTest { + + private static String transportMode = "binary"; + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Set up the base class + ThriftCLIServiceTest.setUpBeforeClass(); + + assertNotNull(port); + assertNotNull(hiveServer2); + assertNotNull(hiveConf); + + hiveConf.setBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + hiveConf.setVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, host); + hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, port); + hiveConf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, AuthTypes.NONE.toString()); + hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, transportMode); + + startHiveServer2WithConf(hiveConf); + + client = getServiceClientInternal(); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + ThriftCLIServiceTest.tearDownAfterClass(); + } + +} \ No newline at end of file diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithHttp.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithHttp.java new file mode 100644 index 0000000..b1c6b67 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithHttp.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.service.cli.thrift; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.jdbc.HttpBasicAuthInterceptor; +import org.apache.hive.service.auth.HiveAuthFactory; +import org.apache.hive.service.auth.HiveAuthFactory.AuthTypes; +import org.apache.hive.service.rpc.thrift.TCLIService; +import org.apache.hive.service.rpc.thrift.TOpenSessionReq; +import org.apache.http.Header; +import org.apache.http.HttpException; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.client.CookieStore; +import org.apache.http.client.protocol.RequestDefaultHeaders; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.message.BasicHeader; +import org.apache.http.protocol.HttpContext; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.THttpClient; +import org.apache.thrift.transport.TTransport; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * + * TestThriftHttpCLIService. + * This tests ThriftCLIService started in http mode. + * + */ + +public class TestThriftCLIServiceWithHttp extends ThriftCLIServiceTest { + + private static String transportMode = "http"; + private static String thriftHttpPath = "cliservice"; + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Set up the base class + ThriftCLIServiceTest.setUpBeforeClass(); + + assertNotNull(port); + assertNotNull(hiveServer2); + assertNotNull(hiveConf); + + hiveConf.setBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + hiveConf.setVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, host); + hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, port); + hiveConf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, AuthTypes.NOSASL.toString()); + hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, transportMode); + hiveConf.setVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH, thriftHttpPath); + + startHiveServer2WithConf(hiveConf); + + client = getServiceClientInternal(); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + ThriftCLIServiceTest.tearDownAfterClass(); + } + + +} diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftHttpCLIService.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftHttpCLIService.java deleted file mode 100644 index 3ed6dd8..0000000 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftHttpCLIService.java +++ /dev/null @@ -1,241 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hive.service.cli.thrift; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; - -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hive.jdbc.HttpBasicAuthInterceptor; -import org.apache.hive.service.auth.HiveAuthFactory; -import org.apache.hive.service.auth.HiveAuthFactory.AuthTypes; -import org.apache.hive.service.rpc.thrift.TCLIService; -import org.apache.hive.service.rpc.thrift.TOpenSessionReq; -import org.apache.http.HttpException; -import org.apache.http.HttpRequest; -import org.apache.http.client.CookieStore; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.protocol.HttpContext; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.transport.THttpClient; -import org.apache.thrift.transport.TTransport; -import org.junit.After; -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * - * TestThriftHttpCLIService. - * This tests ThriftCLIService started in http mode. - * - */ - -public class TestThriftHttpCLIService extends ThriftCLIServiceTest { - - private static String transportMode = "http"; - private static String thriftHttpPath = "cliservice"; - - /** - * HttpBasicAuthInterceptorWithLogging - * This adds httpRequestHeaders to the BasicAuthInterceptor - */ - public class HttpBasicAuthInterceptorWithLogging extends HttpBasicAuthInterceptor { - - ArrayList requestHeaders; - - public HttpBasicAuthInterceptorWithLogging(String username, - String password, CookieStore cookieStore, String cn, boolean isSSL, - Map additionalHeaders) { - super(username, password, cookieStore, cn, isSSL, additionalHeaders); - requestHeaders = new ArrayList(); - } - - @Override - public void process(HttpRequest httpRequest, HttpContext httpContext) - throws HttpException, IOException { - super.process(httpRequest, httpContext); - - String currHeaders = ""; - - for (org.apache.http.Header h : httpRequest.getAllHeaders()) { - currHeaders += h.getName() + ":" + h.getValue() + " "; - } - requestHeaders.add(currHeaders); - } - - public ArrayList getRequestHeaders() { - return requestHeaders; - } - } - - /** - * @throws java.lang.Exception - */ - @BeforeClass - public static void setUpBeforeClass() throws Exception { - // Set up the base class - ThriftCLIServiceTest.setUpBeforeClass(); - - assertNotNull(port); - assertNotNull(hiveServer2); - assertNotNull(hiveConf); - - hiveConf.setBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); - hiveConf.setVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, host); - hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, port); - hiveConf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, AuthTypes.NOSASL.toString()); - hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, transportMode); - hiveConf.setVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH, thriftHttpPath); - - startHiveServer2WithConf(hiveConf); - - client = getServiceClientInternal(); - } - - /** - * @throws java.lang.Exception - */ - @AfterClass - public static void tearDownAfterClass() throws Exception { - ThriftCLIServiceTest.tearDownAfterClass(); - } - - /** - * @throws java.lang.Exception - */ - @Override - @Before - public void setUp() throws Exception { - - } - - /** - * @throws java.lang.Exception - */ - @Override - @After - public void tearDown() throws Exception { - - } - - @Test - /** - * Tests calls from a raw (NOSASL) binary client, - * to a HiveServer2 running in http mode. - * This should throw an expected exception due to incompatibility. - * @throws Exception - */ - public void testBinaryClientHttpServer() throws Exception { - TTransport transport = getRawBinaryTransport(); - TCLIService.Client rawBinaryClient = getClient(transport); - - // This will throw an expected exception since client-server modes are incompatible - testOpenSessionExpectedException(rawBinaryClient); - } - - /** - * Configure a wrong service endpoint for the client transport, - * and test for error. - * @throws Exception - */ - @Test - public void testIncorrectHttpPath() throws Exception { - thriftHttpPath = "wrongPath"; - TTransport transport = getHttpTransport(); - TCLIService.Client httpClient = getClient(transport); - - // This will throw an expected exception since - // client is communicating with the wrong http service endpoint - testOpenSessionExpectedException(httpClient); - - // Reset to correct http path - thriftHttpPath = "cliservice"; - } - - private void testOpenSessionExpectedException(TCLIService.Client client) { - boolean caughtEx = false; - // Create a new open session request object - TOpenSessionReq openReq = new TOpenSessionReq(); - try { - client.OpenSession(openReq).getSessionHandle(); - } catch (Exception e) { - caughtEx = true; - System.out.println("Exception expected: " + e.toString()); - } - assertTrue("Exception expected", caughtEx); - } - - private TCLIService.Client getClient(TTransport transport) throws Exception { - // Create the corresponding client - TProtocol protocol = new TBinaryProtocol(transport); - return new TCLIService.Client(protocol); - } - - private TTransport getRawBinaryTransport() throws Exception { - return HiveAuthFactory.getSocketTransport(host, port, 0); - } - - private static TTransport getHttpTransport() throws Exception { - DefaultHttpClient httpClient = new DefaultHttpClient(); - String httpUrl = transportMode + "://" + host + ":" + port + - "/" + thriftHttpPath + "/"; - httpClient.addRequestInterceptor( - new HttpBasicAuthInterceptor(USERNAME, PASSWORD, null, null, false, null)); - return new THttpClient(httpUrl, httpClient); - } - - /** - * Test additional http headers passed to request interceptor. - * @throws Exception - */ - @Test - public void testAdditionalHttpHeaders() throws Exception { - TTransport transport; - DefaultHttpClient hClient = new DefaultHttpClient(); - String httpUrl = transportMode + "://" + host + ":" + port + - "/" + thriftHttpPath + "/"; - Map additionalHeaders = new HashMap(); - additionalHeaders.put("key1", "value1"); - additionalHeaders.put("key2", "value2"); - HttpBasicAuthInterceptorWithLogging authInt = - new HttpBasicAuthInterceptorWithLogging(USERNAME, PASSWORD, null, null, - false, additionalHeaders); - hClient.addRequestInterceptor(authInt); - transport = new THttpClient(httpUrl, hClient); - TCLIService.Client httpClient = getClient(transport); - - // Create a new open session request object - TOpenSessionReq openReq = new TOpenSessionReq(); - httpClient.OpenSession(openReq).getSessionHandle(); - ArrayList headers = authInt.getRequestHeaders(); - - for (String h : headers) { - assertTrue(h.contains("key1:value1")); - assertTrue(h.contains("key2:value2")); - } - } -} diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftHttpCLIServiceFeatures.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftHttpCLIServiceFeatures.java new file mode 100644 index 0000000..8ad3ae9 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftHttpCLIServiceFeatures.java @@ -0,0 +1,334 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.service.cli.thrift; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.verify; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider; +import org.apache.hadoop.hive.ql.security.SessionStateUserAuthenticator; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizer; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizerFactory; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzSessionContext; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveMetastoreClientFactory; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveOperationType; +import org.apache.hadoop.hive.ql.security.authorization.plugin.HivePrivilegeObject; +import org.apache.hadoop.hive.ql.security.authorization.plugin.QueryContext; +import org.apache.hive.jdbc.HttpBasicAuthInterceptor; +import org.apache.hive.service.auth.HiveAuthFactory; +import org.apache.hive.service.auth.HiveAuthFactory.AuthTypes; +import org.apache.hive.service.rpc.thrift.TCLIService; +import org.apache.hive.service.rpc.thrift.TExecuteStatementReq; +import org.apache.hive.service.rpc.thrift.TOpenSessionReq; +import org.apache.hive.service.rpc.thrift.TOpenSessionResp; +import org.apache.http.Header; +import org.apache.http.HttpException; +import org.apache.http.HttpRequest; +import org.apache.http.client.CookieStore; +import org.apache.http.client.protocol.RequestDefaultHeaders; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.message.BasicHeader; +import org.apache.http.protocol.HttpContext; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.THttpClient; +import org.apache.thrift.transport.TTransport; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Matchers; +import org.mockito.Mockito; + +import com.google.common.base.Joiner; + +/** + * + * Tests that are specific to HTTP transport mode, that need use of underlying + * classes instead of jdbc. + */ + +public class TestThriftHttpCLIServiceFeatures { + + private static String transportMode = "http"; + private static String thriftHttpPath = "cliservice"; + static HiveAuthorizer mockedAuthorizer; + + /** + * HttpBasicAuthInterceptorWithLogging + * This adds httpRequestHeaders to the BasicAuthInterceptor + */ + public class HttpBasicAuthInterceptorWithLogging extends HttpBasicAuthInterceptor { + + ArrayList requestHeaders; + + public HttpBasicAuthInterceptorWithLogging(String username, + String password, CookieStore cookieStore, String cn, boolean isSSL, + Map additionalHeaders) { + super(username, password, cookieStore, cn, isSSL, additionalHeaders); + requestHeaders = new ArrayList(); + } + + @Override + public void process(HttpRequest httpRequest, HttpContext httpContext) + throws HttpException, IOException { + super.process(httpRequest, httpContext); + + String currHeaders = ""; + + for (org.apache.http.Header h : httpRequest.getAllHeaders()) { + currHeaders += h.getName() + ":" + h.getValue() + " "; + } + requestHeaders.add(currHeaders); + } + + public ArrayList getRequestHeaders() { + return requestHeaders; + } + } + + + /** + * @throws java.lang.Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Set up the base class + ThriftCLIServiceTest.setUpBeforeClass(); + + assertNotNull(ThriftCLIServiceTest.port); + assertNotNull(ThriftCLIServiceTest.hiveServer2); + assertNotNull(ThriftCLIServiceTest.hiveConf); + HiveConf hiveConf = ThriftCLIServiceTest.hiveConf; + + hiveConf.setBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + hiveConf.setVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, ThriftCLIServiceTest.host); + hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, ThriftCLIServiceTest.port); + hiveConf.setVar(ConfVars.HIVE_SERVER2_AUTHENTICATION, AuthTypes.NOSASL.toString()); + hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, transportMode); + hiveConf.setVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH, thriftHttpPath); + hiveConf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + + hiveConf.setVar(ConfVars.HIVE_AUTHORIZATION_MANAGER, MockedHiveAuthorizerFactory.class.getName()); + hiveConf.setVar(ConfVars.HIVE_AUTHENTICATOR_MANAGER, SessionStateUserAuthenticator.class.getName()); + hiveConf.setBoolVar(ConfVars.HIVE_AUTHORIZATION_ENABLED, true); + + ThriftCLIServiceTest.startHiveServer2WithConf(hiveConf); + + ThriftCLIServiceTest.client = ThriftCLIServiceTest.getServiceClientInternal(); + } + + /** + * @throws java.lang.Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + ThriftCLIServiceTest.tearDownAfterClass(); + } + + + @Test + /** + * Tests calls from a raw (NOSASL) binary client, + * to a HiveServer2 running in http mode. + * This should throw an expected exception due to incompatibility. + * @throws Exception + */ + public void testBinaryClientHttpServer() throws Exception { + TTransport transport = getRawBinaryTransport(); + TCLIService.Client rawBinaryClient = getClient(transport); + + // This will throw an expected exception since client-server modes are incompatible + testOpenSessionExpectedException(rawBinaryClient); + } + + /** + * Configure a wrong service endpoint for the client transport, + * and test for error. + * @throws Exception + */ + @Test + public void testIncorrectHttpPath() throws Exception { + thriftHttpPath = "wrongPath"; + TTransport transport = getHttpTransport(); + TCLIService.Client httpClient = getClient(transport); + + // This will throw an expected exception since + // client is communicating with the wrong http service endpoint + testOpenSessionExpectedException(httpClient); + + // Reset to correct http path + thriftHttpPath = "cliservice"; + } + + private void testOpenSessionExpectedException(TCLIService.Client client) { + boolean caughtEx = false; + // Create a new open session request object + TOpenSessionReq openReq = new TOpenSessionReq(); + try { + client.OpenSession(openReq).getSessionHandle(); + } catch (Exception e) { + caughtEx = true; + System.out.println("Exception expected: " + e.toString()); + } + assertTrue("Exception expected", caughtEx); + } + + private TCLIService.Client getClient(TTransport transport) throws Exception { + // Create the corresponding client + TProtocol protocol = new TBinaryProtocol(transport); + return new TCLIService.Client(protocol); + } + + private TTransport getRawBinaryTransport() throws Exception { + return HiveAuthFactory.getSocketTransport(ThriftCLIServiceTest.host, ThriftCLIServiceTest.port, 0); + } + + private static TTransport getHttpTransport() throws Exception { + DefaultHttpClient httpClient = new DefaultHttpClient(); + String httpUrl = getHttpUrl(); + httpClient.addRequestInterceptor( + new HttpBasicAuthInterceptor(ThriftCLIServiceTest.USERNAME, ThriftCLIServiceTest.PASSWORD, + null, null, false, null)); + return new THttpClient(httpUrl, httpClient); + } + + private static String getHttpUrl() { + return transportMode + "://" + ThriftCLIServiceTest.host + ":" + + ThriftCLIServiceTest.port + + "/" + thriftHttpPath + "/"; + } + + /** + * Test additional http headers passed to request interceptor. + * @throws Exception + */ + @Test + public void testAdditionalHttpHeaders() throws Exception { + TTransport transport; + DefaultHttpClient hClient = new DefaultHttpClient(); + String httpUrl = getHttpUrl(); + Map additionalHeaders = new HashMap(); + additionalHeaders.put("key1", "value1"); + additionalHeaders.put("key2", "value2"); + HttpBasicAuthInterceptorWithLogging authInt = + new HttpBasicAuthInterceptorWithLogging(ThriftCLIServiceTest.USERNAME, ThriftCLIServiceTest.PASSWORD, null, null, + false, additionalHeaders); + hClient.addRequestInterceptor(authInt); + transport = new THttpClient(httpUrl, hClient); + TCLIService.Client httpClient = getClient(transport); + + // Create a new open session request object + TOpenSessionReq openReq = new TOpenSessionReq(); + httpClient.OpenSession(openReq).getSessionHandle(); + ArrayList headers = authInt.getRequestHeaders(); + + for (String h : headers) { + assertTrue(h.contains("key1:value1")); + assertTrue(h.contains("key2:value2")); + } + } + + /** + * This factory creates a mocked HiveAuthorizer class. + * Use the mocked class to capture the argument passed to it in the test case. + */ + static class MockedHiveAuthorizerFactory implements HiveAuthorizerFactory { + @Override + public HiveAuthorizer createHiveAuthorizer(HiveMetastoreClientFactory metastoreClientFactory, + HiveConf conf, HiveAuthenticationProvider authenticator, HiveAuthzSessionContext ctx) { + mockedAuthorizer = Mockito.mock(HiveAuthorizer.class); + return mockedAuthorizer; + } + } + + /** + * Test if addresses in X-Forwarded-For are passed to HiveAuthorizer calls + * @throws Exception + */ + @Test + public void testForwardedHeaders() throws Exception { + verifyForwardedHeaders(new ArrayList(Arrays.asList("127.0.0.1", "202.101.101.101")), "show tables"); + verifyForwardedHeaders(new ArrayList(Arrays.asList("202.101.101.101")), "fs -ls /"); + verifyForwardedHeaders(new ArrayList(), "show databases"); + } + + private void verifyForwardedHeaders(ArrayList headerIPs, String cmd) throws Exception { + TTransport transport; + DefaultHttpClient hClient = new DefaultHttpClient(); + String httpUrl = getHttpUrl(); + + // add an interceptor that adds the X-Forwarded-For header with given ips + if (!headerIPs.isEmpty()) { + Header xForwardHeader = new BasicHeader("X-Forwarded-For", Joiner.on(",").join(headerIPs)); + RequestDefaultHeaders headerInterceptor = new RequestDefaultHeaders( + Arrays.asList(xForwardHeader)); + hClient.addRequestInterceptor(headerInterceptor); + } + + // interceptor for adding username, pwd + HttpBasicAuthInterceptor authInt = new HttpBasicAuthInterceptor(ThriftCLIServiceTest.USERNAME, + ThriftCLIServiceTest.PASSWORD, null, null, + false, null); + hClient.addRequestInterceptor(authInt); + + transport = new THttpClient(httpUrl, hClient); + TCLIService.Client httpClient = getClient(transport); + + // Create a new open session request object + TOpenSessionReq openReq = new TOpenSessionReq(); + TOpenSessionResp openResp = httpClient.OpenSession(openReq); + + //execute a query + TExecuteStatementReq execReq = new TExecuteStatementReq(openResp.getSessionHandle(), "show tables"); + httpClient.ExecuteStatement(execReq); + + // capture arguments to authorizer impl call and verify ip addresses passed + ArgumentCaptor contextCapturer = ArgumentCaptor + .forClass(QueryContext.class); + + verify(mockedAuthorizer).checkPrivileges(any(HiveOperationType.class), + Matchers.anyListOf(HivePrivilegeObject.class), + Matchers.anyListOf(HivePrivilegeObject.class), contextCapturer.capture()); + + QueryContext context = contextCapturer.getValue(); + System.err.println("Forwarded IP Addresses " + context.getForwardedAddresses()); + + List auditIPAddresses = new ArrayList(context.getForwardedAddresses()); + Collections.sort(auditIPAddresses); + Collections.sort(headerIPs); + + Assert.assertEquals("Checking forwarded IP Address" , headerIPs, auditIPAddresses); + } + + +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 65ed1db..33ac21c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -848,6 +848,7 @@ private static void doAuthorizationV2(SessionState ss, HiveOperation op, Set getFilteredObjects(List listObjs) throws MetaException { SessionState ss = SessionState.get(); QueryContext.Builder authzContextBuilder = new QueryContext.Builder(); + authzContextBuilder.setForwardedAddresses(ss.getForwardedAddresses()); try { return ss.getAuthorizerV2().filterListCmdObjects(listObjs, authzContextBuilder.build()); } catch (HiveAuthzPluginException e) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/QueryContext.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/QueryContext.java index 318343c..17f8913 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/QueryContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/QueryContext.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.ql.security.authorization.plugin; +import java.util.List; + import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.hive.common.classification.InterfaceStability.Evolving; @@ -31,6 +33,7 @@ public static class Builder { private String commandString; + private List forwardedAddresses; public String getCommandString() { return commandString; @@ -38,24 +41,38 @@ public String getCommandString() { public void setCommandString(String commandString) { this.commandString = commandString; } + + public List getForwardedAddresses() { + return forwardedAddresses; + } + public void setForwardedAddresses(List forwardedAddresses) { + this.forwardedAddresses = forwardedAddresses; + } + public QueryContext build(){ return new QueryContext(this); } } private final String commandString; + private final List forwardedAddresses; private QueryContext(Builder builder) { this.commandString = builder.commandString; + this.forwardedAddresses = builder.forwardedAddresses; } public String getCommandString() { return commandString; } + public List getForwardedAddresses() { + return forwardedAddresses; + } + @Override public String toString() { - return "QueryContext [commandString=" + commandString + "]"; + return "QueryContext [commandString=" + commandString + ", forwardedAddresses=" + forwardedAddresses + "]"; } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 23b8a96..cc3393c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -269,6 +269,8 @@ private final ResourceDownloader resourceDownloader; + private List forwardedAddresses; + /** * Get the lineage state stored in this session. * @@ -1660,6 +1662,14 @@ public Timestamp getQueryCurrentTimestamp() { public ResourceDownloader getResourceDownloader() { return resourceDownloader; } + + public void setForwardedAddresses(List forwardedAddresses) { + this.forwardedAddresses = forwardedAddresses; + } + + public List getForwardedAddresses() { + return forwardedAddresses; + } } class ResourceMaps { diff --git a/service/src/java/org/apache/hive/service/cli/operation/MetadataOperation.java b/service/src/java/org/apache/hive/service/cli/operation/MetadataOperation.java index c4a7e69..77228fa 100644 --- a/service/src/java/org/apache/hive/service/cli/operation/MetadataOperation.java +++ b/service/src/java/org/apache/hive/service/cli/operation/MetadataOperation.java @@ -135,6 +135,7 @@ protected void authorizeMetaGets(HiveOperationType opType, List sessionConfMap) throws HiveSQLException { sessionState = new SessionState(hiveConf, username); sessionState.setUserIpAddress(ipAddress); sessionState.setIsHiveServerQuery(true); + sessionState.setForwardedAddresses(SessionManager.getForwardedAddresses()); SessionState.start(sessionState); try { sessionState.reloadAuxJars(); @@ -326,6 +327,7 @@ protected synchronized void acquire(boolean userAccess) { // Need to make sure that the this HiveServer2's session's SessionState is // stored in the thread local for the handler thread. SessionState.setCurrentSessionState(sessionState); + sessionState.setForwardedAddresses(SessionManager.getForwardedAddresses()); if (userAccess) { lastAccessTime = System.currentTimeMillis(); } diff --git a/service/src/java/org/apache/hive/service/cli/session/SessionManager.java b/service/src/java/org/apache/hive/service/cli/session/SessionManager.java index 79eddd9..ad8678e 100644 --- a/service/src/java/org/apache/hive/service/cli/session/SessionManager.java +++ b/service/src/java/org/apache/hive/service/cli/session/SessionManager.java @@ -395,12 +395,7 @@ public OperationManager getOperationManager() { return operationManager; } - private static ThreadLocal threadLocalIpAddress = new ThreadLocal() { - @Override - protected String initialValue() { - return null; - } - }; + private static ThreadLocal threadLocalIpAddress = new ThreadLocal(); public static void setIpAddress(String ipAddress) { threadLocalIpAddress.set(ipAddress); @@ -414,6 +409,20 @@ public static String getIpAddress() { return threadLocalIpAddress.get(); } + private static ThreadLocal> threadLocalForwardedAddresses = new ThreadLocal>(); + + public static void setForwardedAddresses(List ipAddress) { + threadLocalForwardedAddresses.set(ipAddress); + } + + public static void clearForwardedAddresses() { + threadLocalForwardedAddresses.remove(); + } + + public static List getForwardedAddresses() { + return threadLocalForwardedAddresses.get(); + } + private static ThreadLocal threadLocalUserName = new ThreadLocal(){ @Override protected String initialValue() { diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java index 7e12fae..74d73b7 100644 --- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java +++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpServlet.java @@ -20,7 +20,11 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.net.InetAddress; import java.security.PrivilegedExceptionAction; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; @@ -87,6 +91,7 @@ private boolean isHttpOnlyCookie; private final HiveAuthFactory hiveAuthFactory; private static final String HIVE_DELEGATION_TOKEN_HEADER = "X-Hive-Delegation-Token"; + private static final String X_FORWARDED_FOR = "X-Forwarded-For"; public ThriftHttpServlet(TProcessor processor, TProtocolFactory protocolFactory, String authType, UserGroupInformation serviceUGI, UserGroupInformation httpUGI, @@ -166,6 +171,17 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) LOG.debug("Client IP Address: " + clientIpAddress); // Set the thread local ip address SessionManager.setIpAddress(clientIpAddress); + + // get forwarded hosts address + String forwarded_for = request.getHeader(X_FORWARDED_FOR); + if (forwarded_for != null) { + LOG.debug("{}:{}", X_FORWARDED_FOR, forwarded_for); + List forwardedAddresses = Arrays.asList(forwarded_for.split(",")); + SessionManager.setForwardedAddresses(forwardedAddresses); + } else { + SessionManager.setForwardedAddresses(Collections.emptyList()); + } + // Generate new cookie and add it to the response if (requireNewCookie && !authType.equalsIgnoreCase(HiveAuthFactory.AuthTypes.NOSASL.toString())) { @@ -195,6 +211,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) SessionManager.clearUserName(); SessionManager.clearIpAddress(); SessionManager.clearProxyUserName(); + SessionManager.clearForwardedAddresses(); } } diff --git a/service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java b/service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java index 630cfc9..1740079 100644 --- a/service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java +++ b/service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java @@ -73,7 +73,7 @@ public static void tearDownAfterClass() throws Exception { stopHiveServer2(); } - protected static void startHiveServer2WithConf(HiveConf hiveConf) throws Exception { + static void startHiveServer2WithConf(HiveConf hiveConf) throws Exception { hiveServer2.init(hiveConf); // Start HiveServer2 with given config // Fail if server doesn't start @@ -94,7 +94,7 @@ protected static void stopHiveServer2() throws Exception { } } - protected static ThriftCLIServiceClient getServiceClientInternal() { + static ThriftCLIServiceClient getServiceClientInternal() { for (Service service : hiveServer2.getServices()) { if (service instanceof ThriftBinaryCLIService) { return new ThriftCLIServiceClient((ThriftBinaryCLIService) service);