diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 73f185a1f3bf6b299349e90fdc497b8a7ba0a708..4ba53106e23b6dd6662e494d4ba2e38a7e8c6b21 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3329,7 +3329,10 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION("hive.server2.global.init.file.location", "${env:HIVE_CONF_DIR}", "Either the location of a HS2 global init file or a directory containing a .hiverc file. If the \n" + "property is set, the value must be a valid path to an init file or directory where the init file is located."), - HIVE_SERVER2_TRANSPORT_MODE("hive.server2.transport.mode", "binary", new StringSet("binary", "http"), + HIVE_SERVER2_TRANSPORT_MODE("hive.server2.transport.mode", + HiveServer2TransportMode.binary.toString(), + new StringSet(HiveServer2TransportMode.binary.toString(), + HiveServer2TransportMode.http.toString(), HiveServer2TransportMode.all.toString()), "Transport mode of HiveServer2."), HIVE_SERVER2_THRIFT_BIND_HOST("hive.server2.thrift.bind.host", "", "Bind host on which to run the HiveServer2 Thrift service."), diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveServer2TransportMode.java b/common/src/java/org/apache/hadoop/hive/conf/HiveServer2TransportMode.java new file mode 100644 index 0000000000000000000000000000000000000000..61456517651d914ce66c8c784a80be87338c12c7 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveServer2TransportMode.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.conf; + +/** + * Hive Transport mode. + */ +public enum HiveServer2TransportMode { + /** + * Three modes: http, binary or all (which includes binary as well as http). + */ + http, binary, all +} diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java index dc088930a1420ce108702828d916da51900c3a37..5d528a57bf7c5629f19bdf5229ce0948ffed0524 100644 --- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java @@ -194,7 +194,7 @@ public static MiniHS2 getMiniHS2WithKerb(MiniHiveKdc miniHiveKdc, HiveConf hiveC .withConf(hiveConf) .withMiniKdc(hivePrincipal, hiveKeytab) .withAuthenticationType(authType); - if (HiveServer2.isHTTPTransportMode(hiveConf)) { + if (HiveServer2.isHttpTransportMode(hiveConf)) { miniHS2Builder.withHTTPTransport(); } return miniHS2Builder.build(); diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcCookie.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcCookie.java index 2fa2a876040821f82d53cf5ee1e84be1fb45358c..33189202abf92fb4bd65a6db70fd386f32ad1109 100644 --- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcCookie.java +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcCookie.java @@ -21,7 +21,10 @@ import java.io.File; import java.io.IOException; import java.sql.Connection; +import java.sql.DriverManager; import java.sql.Statement; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.Properties; import java.util.concurrent.TimeUnit; @@ -30,56 +33,68 @@ import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hive.jdbc.HiveConnection; import org.apache.hive.jdbc.miniHS2.MiniHS2; + import org.junit.After; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +/** + * Testing JDBC with Mini KDC. + */ +@RunWith(Parameterized.class) public class TestJdbcWithMiniKdcCookie { private static MiniHS2 miniHS2 = null; - private static MiniHiveKdc miniHiveKdc = null; - private Connection hs2Conn; + private MiniHiveKdc miniHiveKdc = null; + private static Connection hs2Conn; File dataFile; protected static HiveConf hiveConf; private static String HIVE_NON_EXISTENT_USER = "hive_no_exist"; + @Parameterized.Parameter + public String transportMode = null; + + @Parameterized.Parameters(name = "{index}: tranportMode={0}") + public static Collection transportModes() { + return Arrays.asList(new Object[][]{{MiniHS2.HS2_ALL_MODE}, {MiniHS2.HS2_HTTP_MODE}}); + } + @BeforeClass public static void beforeTest() throws Exception { + Class.forName(MiniHS2.getJdbcDriverName()); + } + + @Before + public void setUp() throws Exception { miniHiveKdc = new MiniHiveKdc(); + DriverManager.setLoginTimeout(0); hiveConf = new HiveConf(); - hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, MiniHS2.HS2_HTTP_MODE); + hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, transportMode); System.err.println("Testing using HS2 mode : " - + hiveConf.getVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE)); + + hiveConf.getVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE)); hiveConf.setBoolVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_AUTH_ENABLED, - true); + true); // set a small time unit as cookie max age so that the server sends a 401 hiveConf.setTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_MAX_AGE, - 1, TimeUnit.SECONDS); + 1, TimeUnit.SECONDS); hiveConf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); miniHS2 = MiniHiveKdc.getMiniHS2WithKerb(miniHiveKdc, hiveConf); miniHS2.start(new HashMap()); } - @Before - public void setUp() throws Exception { - } - @After public void tearDown() throws Exception { if (hs2Conn != null) { - try { - hs2Conn.close(); - } catch (Exception e) { - // Ignore shutdown errors since there are negative tests - } + hs2Conn.close(); + hs2Conn = null; + } + if (miniHS2 != null && miniHS2.isStarted()) { + miniHS2.stop(); + miniHS2.cleanup(); } - } - - @AfterClass - public static void afterTest() throws Exception { - miniHS2.stop(); } @Test @@ -100,9 +115,10 @@ public void testCookie() throws Exception { } stmt.execute("drop table " + tableName); stmt.close(); + + testCookieNegative(); } - @Test public void testCookieNegative() throws Exception { try { // Trying to connect with a non-existent user should still fail with @@ -115,6 +131,6 @@ public void testCookieNegative() throws Exception { private Connection getConnection(String userName) throws Exception { miniHiveKdc.loginUser(userName); - return new HiveConnection(miniHS2.getJdbcURL(), new Properties()); + return new HiveConnection(miniHS2.getHttpJdbcURL(), new Properties()); } } diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcSQLAuthAll.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcSQLAuthAll.java new file mode 100644 index 0000000000000000000000000000000000000000..18a1e20648f05100cd7d6149aa089a8fbdfeab34 --- /dev/null +++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcSQLAuthAll.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.minikdc; + +import org.apache.hive.jdbc.miniHS2.MiniHS2; + +import org.junit.BeforeClass; + +/** + * JdbcWithMiniKdcSQLAuthTest for the case of hive.server2.transport.mode=all. + */ +public class TestJdbcWithMiniKdcSQLAuthAll extends JdbcWithMiniKdcSQLAuthTest { + + @BeforeClass + public static void beforeTest() throws Exception { + JdbcWithMiniKdcSQLAuthTest.beforeTestBase(MiniHS2.HS2_ALL_MODE); + } +} diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/BeelineWithHS2ConnectionFileTestBase.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/BeelineWithHS2ConnectionFileTestBase.java index 06ada233a273f4882bab5fc5a573ed9fc2f7edff..13679867c1b28a2f85b560279e7d143136f1436d 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/BeelineWithHS2ConnectionFileTestBase.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/BeelineWithHS2ConnectionFileTestBase.java @@ -47,7 +47,13 @@ import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +/** + * BeelineWithHS2ConnectionFileTestBase test. + */ +@RunWith(Parameterized.class) public abstract class BeelineWithHS2ConnectionFileTestBase { protected MiniHS2 miniHS2; protected HiveConf hiveConf = new HiveConf(); @@ -65,6 +71,10 @@ protected Map confOverlay = new HashMap<>(); + @Parameterized.Parameter + public String transportMode = null; + + protected class TestBeeLine extends BeeLine { UserHS2ConnectionFileParser testHs2ConfigFileManager; ByteArrayOutputStream os; @@ -163,7 +173,8 @@ public void before() throws Exception { miniHS2 = getNewMiniHS2(); confOverlay = new HashMap(); confOverlay.put(ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); - confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, "binary"); + confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, transportMode); + confOverlay.put(ConfVars.HIVE_SERVER2_USE_SSL.varname, "false"); } protected MiniHS2 getNewMiniHS2() throws Exception { diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java index 480d414776525a09d2749bb92ced516b028f3886..86de2e43f8c7dfdd9b466a7058d14fcb68843036 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java @@ -22,20 +22,43 @@ import static org.junit.Assert.assertTrue; import java.io.File; +import java.util.Arrays; +import java.util.Collection; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.junit.Assume; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +/** + * TestBeelineConnectionUsingHiveSite test. + */ +@RunWith(Parameterized.class) public class TestBeelineConnectionUsingHiveSite extends BeelineWithHS2ConnectionFileTestBase { + + boolean isHttpModeTest = false; + + @Parameterized.Parameters(name = "{index}: tranportMode={0}") + public static Collection transportModes() { + return Arrays.asList(new Object[][]{{MiniHS2.HS2_BINARY_MODE}, {MiniHS2.HS2_HTTP_MODE}, {MiniHS2.HS2_ALL_MODE}}); + } @Test public void testBeelineConnectionHttp() throws Exception { + Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_HTTP_MODE) + || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE)); + isHttpModeTest = true; setupHs2(); String path = createDefaultHs2ConnectionFile(); assertBeelineOutputContains(path, new String[] { "-e", "show tables;" }, tableName); + isHttpModeTest = false; } @Test public void testBeelineConnectionSSL() throws Exception { + Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_BINARY_MODE) + || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE)); setupSSLHs2(); String path = createDefaultHs2ConnectionFile(); assertBeelineOutputContains(path, new String[] { "-e", "show tables;" }, tableName); @@ -43,6 +66,8 @@ public void testBeelineConnectionSSL() throws Exception { @Test public void testBeelineConnectionNoAuth() throws Exception { + Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_BINARY_MODE) + || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE)); setupNoAuthHs2(); String path = createDefaultHs2ConnectionFile(); assertBeelineOutputContains(path, new String[] { "-e", "show tables;" }, tableName); @@ -64,6 +89,8 @@ public void testBeelineDoesntUseDefaultIfU() throws Exception { */ @Test public void testBeelineWithNoConnectionFile() throws Exception { + Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_BINARY_MODE) + || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE)); setupNoAuthHs2(); BeelineResult res = getBeelineOutput(null, new String[] {"-e", "show tables;" }); assertEquals(1, res.exitCode); @@ -72,6 +99,8 @@ public void testBeelineWithNoConnectionFile() throws Exception { @Test public void testBeelineUsingArgs() throws Exception { + Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_BINARY_MODE) + || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE)); setupNoAuthHs2(); String url = miniHS2.getBaseJdbcURL() + "default"; String args[] = new String[] { "-u", url, "-n", System.getProperty("user.name"), "-p", "foo", @@ -98,8 +127,6 @@ private void setupSSLHs2() throws Exception { } private void setupHs2() throws Exception { - confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_HTTP_MODE); - confOverlay.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, HS2_HTTP_ENDPOINT); confOverlay.put(ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true"); miniHS2.start(confOverlay); createTable(); @@ -108,6 +135,9 @@ private void setupHs2() throws Exception { private String createDefaultHs2ConnectionFile() throws Exception { Hs2ConnectionXmlConfigFileWriter writer = new Hs2ConnectionXmlConfigFileWriter(); String baseJdbcURL = miniHS2.getBaseJdbcURL(); + if(isHttpModeTest) { + baseJdbcURL = miniHS2.getBaseHttpJdbcURL(); + } System.out.println(baseJdbcURL); writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "user", System.getProperty("user.name")); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithUserHs2ConnectionFile.java b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithUserHs2ConnectionFile.java index 84a8b3754b0be8e7aaef085d7f054d63c6d7e9f4..5de289122b531cf22c6f94fea238330c6a4dfd18 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithUserHs2ConnectionFile.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithUserHs2ConnectionFile.java @@ -19,22 +19,37 @@ import java.io.File; import java.net.URI; +import java.util.Arrays; +import java.util.Collection; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.junit.Assume; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +/** + * TestBeelineWithUserHs2ConnectionFile test. + */ +@RunWith(Parameterized.class) public class TestBeelineWithUserHs2ConnectionFile extends BeelineWithHS2ConnectionFileTestBase { + @Parameterized.Parameters(name = "{index}: tranportMode={0}") + public static Collection transportModes() { + return Arrays.asList(new Object[][]{{MiniHS2.HS2_ALL_MODE}, {MiniHS2.HS2_BINARY_MODE}, {MiniHS2.HS2_HTTP_MODE}}); + } + @Test public void testBeelineConnectionHttp() throws Exception { + Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_HTTP_MODE) + || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE)); setupHttpHs2(); String path = createHttpHs2ConnectionFile(); assertBeelineOutputContains(path, new String[] { "-e", "show tables;" }, tableName); } private void setupHttpHs2() throws Exception { - confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_HTTP_MODE); - confOverlay.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, HS2_HTTP_ENDPOINT); confOverlay.put(ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true"); miniHS2.start(confOverlay); createTable(); @@ -42,7 +57,7 @@ private void setupHttpHs2() throws Exception { private String createHttpHs2ConnectionFile() throws Exception { Hs2ConnectionXmlConfigFileWriter writer = new Hs2ConnectionXmlConfigFileWriter(); - String baseJdbcURL = miniHS2.getBaseJdbcURL(); + String baseJdbcURL = miniHS2.getBaseHttpJdbcURL(); URI uri = new URI(baseJdbcURL.substring(5)); writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "hosts", @@ -62,6 +77,8 @@ private String createHttpHs2ConnectionFile() throws Exception { @Test public void testBeelineConnectionNoAuth() throws Exception { + Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_BINARY_MODE) + || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE)); setupNoAuthConfHS2(); String path = createNoAuthHs2ConnectionFile(); assertBeelineOutputContains(path, new String[] { "-e", "show tables;" }, tableName); @@ -89,6 +106,8 @@ private String createNoAuthHs2ConnectionFile() throws Exception { @Test public void testBeelineConnectionSSL() throws Exception { + Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_BINARY_MODE) + || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE)); setupSslHs2(); String path = createSSLHs2ConnectionFile(); assertBeelineOutputContains(path, new String[] { "-e", "show tables;" }, tableName); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestXSRFFilter.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestXSRFFilter.java index fcabe99a4527c7ac3f00cee3d0245d37a7100ce2..b5a3568ab2cba6cf3001e6131fda6860924c6695 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestXSRFFilter.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestXSRFFilter.java @@ -26,6 +26,8 @@ import java.sql.SQLException; import java.sql.Statement; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -38,10 +40,17 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +/** + * TestXSRFFilter test. + */ +@RunWith(Parameterized.class) public class TestXSRFFilter { private static MiniHS2 miniHS2 = null; @@ -51,6 +60,15 @@ private Connection hs2Conn = null; + @Parameterized.Parameter + public String transportMode = null; + + @Parameterized.Parameters(name = "{index}: tranportMode={0}") + public static Collection transportModes() { + return Arrays.asList(new Object[][]{{MiniHS2.HS2_ALL_MODE}, {MiniHS2.HS2_HTTP_MODE}}); + } + + @BeforeClass public static void beforeClass() throws IOException { MiniHS2.cleanupLocalDir(); @@ -72,7 +90,7 @@ private void initHS2(boolean enableXSRFFilter) throws Exception { kvDataFilePath = new Path(dataFileDir, "kv1.txt"); Map confOverlay = new HashMap(); confOverlay.put(ConfVars.HIVE_SERVER2_XSRF_FILTER_ENABLED.varname, String.valueOf(enableXSRFFilter)); - confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, "http"); + confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, transportMode); miniHS2.start(confOverlay); } @@ -138,7 +156,7 @@ private void runTest(boolean filterEnabled, boolean injectionEnabled) throws Exc private void runBasicCommands() throws Exception { - hs2Conn = getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar"); + hs2Conn = getConnection(miniHS2.getHttpJdbcURL(), System.getProperty("user.name"), "bar"); String tableName = "testTab1"; Statement stmt = hs2Conn.createStatement(); diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithAllandBinary.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithAllandBinary.java new file mode 100644 index 0000000000000000000000000000000000000000..d132520ff8754c263e4be2ccd7e7e905bd28595e --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithAllandBinary.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.service.cli.thrift; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.Service; +import org.apache.hive.service.auth.HiveAuthConstants; + +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import static org.junit.Assert.assertNotNull; + +/** + * TestThriftHttpCLIService. + * This tests ThriftCLIService started in http mode. + */ + +public class TestThriftCLIServiceWithAllandBinary extends ThriftCLIServiceTest { + + private static String transportMode = "all"; + + /** + * @throws Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Set up the base class + ThriftCLIServiceTest.setUpBeforeClass(); + + assertNotNull(port); + assertNotNull(hiveServer2); + assertNotNull(hiveConf); + + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, host); + hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, port); + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION, HiveAuthConstants.AuthTypes.NONE.toString()); + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE, transportMode); + + startHiveServer2WithConf(hiveConf); + + client = getHttpServiceClientInternal(); + } + + /** + * @throws Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + ThriftCLIServiceTest.tearDownAfterClass(); + } + static ThriftCLIServiceClient getHttpServiceClientInternal() { + for (Service service : hiveServer2.getServices()) { + if (service instanceof ThriftBinaryCLIService) { + return new ThriftCLIServiceClient((ThriftBinaryCLIService) service); + } + if (service instanceof ThriftHttpCLIService) { + continue; + } + } + throw new IllegalStateException("HiveServer2 not running Thrift service"); + } +} diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithAllandHttp.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithAllandHttp.java new file mode 100644 index 0000000000000000000000000000000000000000..c91a6b74aa80d0c94a02a15d7348f259f7db4834 --- /dev/null +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithAllandHttp.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.service.cli.thrift; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.Service; +import org.apache.hive.service.auth.HiveAuthConstants; + +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import static org.junit.Assert.assertNotNull; + +/** + * TestThriftHttpCLIService. + * This tests ThriftCLIService started in http mode. + */ + +public class TestThriftCLIServiceWithAllandHttp extends ThriftCLIServiceTest { + + private static String transportMode = "all"; + private static String thriftHttpPath = "cliservice"; + + /** + * @throws Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Set up the base class + ThriftCLIServiceTest.setUpBeforeClass(); + + assertNotNull(port); + assertNotNull(hiveServer2); + assertNotNull(hiveConf); + + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, host); + hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, port); + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION, HiveAuthConstants.AuthTypes.NOSASL.toString()); + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE, transportMode); + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH, thriftHttpPath); + + startHiveServer2WithConf(hiveConf); + + client = getHttpServiceClientInternal(); + } + + /** + * @throws Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + ThriftCLIServiceTest.tearDownAfterClass(); + } + static ThriftCLIServiceClient getHttpServiceClientInternal() { + for (Service service : hiveServer2.getServices()) { + if (service instanceof ThriftBinaryCLIService) { + continue; + } + if (service instanceof ThriftHttpCLIService) { + return new ThriftCLIServiceClient((ThriftHttpCLIService) service); + } + } + throw new IllegalStateException("HiveServer2 not running Thrift service"); + } +} diff --git a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index 1b60a51ebdabff9f0b6e6725d888ba7546d12438..9ae12b28a8c5809bd2be69818167c28b0b5df463 100644 --- a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -62,6 +62,7 @@ public static final String HS2_BINARY_MODE = "binary"; public static final String HS2_HTTP_MODE = "http"; + public static final String HS2_ALL_MODE = "all"; private static final String driverName = "org.apache.hive.jdbc.HiveDriver"; private static final FsPermission FULL_PERM = new FsPermission((short)00777); private static final FsPermission WRITE_ALL_PERM = new FsPermission((short)00733); @@ -566,8 +567,20 @@ public String getBaseJdbcURL() { } /** - * Build zk base JDBC URL - * @return + * Build base JDBC URL + * @return URL + */ + public String getBaseHttpJdbcURL() { + String transportMode = getConfProperty(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname); + if(!transportMode.equalsIgnoreCase(HS2_ALL_MODE)) { + return getBaseJdbcURL(); + } + return "jdbc:hive2://" + getHost() + ":" + getHttpPort() + "/"; + } + + /** + * Build zk base JDBC URL. + * @return URL */ private String getZKBaseJdbcURL() throws Exception { HiveConf hiveConf = getServerConf(); @@ -578,6 +591,23 @@ private String getZKBaseJdbcURL() throws Exception { throw new Exception("Server's HiveConf is null. Unable to read ZooKeeper configs."); } + /** + * Returns HTTP connection URL for this server instance. + * @return URL + * @throws Exception + */ + public String getHttpJdbcURL() throws Exception { + String transportMode = getConfProperty(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname); + if(!transportMode.equalsIgnoreCase(HS2_ALL_MODE)) { + return getJdbcURL(); + } + //CAUTION: This will result in concurrency issues. + getHiveConf().setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_HTTP_MODE); + String url = getJdbcURL("default"); + getHiveConf().setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_ALL_MODE); + return url; + } + private boolean isHttpTransportMode() { String transportMode = getConfProperty(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname); return transportMode != null && (transportMode.equalsIgnoreCase(HS2_HTTP_MODE)); diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java index df2d3a7b71978903fe083c744eb3d4f7a53b1944..dfe99e6b693bb21aefdad62813ac00262e79f4d2 100644 --- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java +++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveServer2TransportMode; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hive.service.auth.HiveAuthFactory; import org.apache.hive.service.cli.CLIService; @@ -58,6 +59,11 @@ public ThriftBinaryCLIService(CLIService cliService, Runnable oomHook) { this.oomHook = oomHook; } + @Override + protected HiveServer2TransportMode getTransportMode() { + return HiveServer2TransportMode.binary; + } + @Override protected void initServer() { try { diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java index 6f8ad00c64b752d3bc1487fac16cd870fd33cf69..b7b93a4e5a4413ab0ce5b769f6b9baeae2c551f4 100644 --- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java +++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hive.common.log.ProgressMonitor; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveServer2TransportMode; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.HadoopShims.KerberosNameShim; import org.apache.hadoop.hive.shims.ShimLoader; @@ -112,10 +113,8 @@ import org.apache.hive.service.rpc.thrift.TRenewDelegationTokenResp; import org.apache.hive.service.rpc.thrift.TStatus; import org.apache.hive.service.rpc.thrift.TStatusCode; -import org.apache.hive.service.server.HiveServer2; import org.apache.thrift.TException; import org.apache.thrift.server.ServerContext; -import org.apache.thrift.server.TServer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -182,7 +181,7 @@ public synchronized void init(HiveConf hiveConf) { // Initialize common server configs needed in both binary & http modes String portString; // HTTP mode - if (HiveServer2.isHTTPTransportMode(hiveConf)) { + if (getTransportMode() == HiveServer2TransportMode.http) { workerKeepAliveTime = hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_WORKER_KEEPALIVE_TIME, TimeUnit.SECONDS); @@ -378,8 +377,10 @@ private String getIpAddress() { String clientIpAddress; // Http transport mode. // We set the thread local ip address, in ThriftHttpServlet. - if (cliService.getHiveConf().getVar( - ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) { + //Since we are allowing multiple transport modes, this + //transport information comes from the subclasses + //instead of reading from hive configuration + if (getTransportMode() == HiveServer2TransportMode.http) { clientIpAddress = SessionManager.getIpAddress(); } else { @@ -417,8 +418,7 @@ private String getUserName(TOpenSessionReq req) throws HiveSQLException, IOExcep } // Http transport mode. // We set the thread local username, in ThriftHttpServlet. - if (cliService.getHiveConf().getVar( - ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) { + if (getTransportMode() == HiveServer2TransportMode.http) { userName = SessionManager.getUserName(); } if (userName == null) { @@ -862,12 +862,17 @@ public TGetQueryIdResp GetQueryId(TGetQueryIdReq req) throws TException { @Override public abstract void run(); + /** Transport mode of hiveserver2 thrift. + * @return the mode. + */ + protected abstract HiveServer2TransportMode getTransportMode(); + /** * If the proxy user name is provided then check privileges to substitute the user. * @param realUser * @param sessionConf * @param ipAddress - * @return + * @return username * @throws HiveSQLException */ private String getProxyUser(String realUser, Map sessionConf, @@ -875,8 +880,7 @@ private String getProxyUser(String realUser, Map sessionConf, String proxyUser = null; // Http transport mode. // We set the thread local proxy username, in ThriftHttpServlet. - if (cliService.getHiveConf().getVar( - ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) { + if (getTransportMode() == HiveServer2TransportMode.http) { proxyUser = SessionManager.getProxyUserName(); LOG.debug("Proxy user from query string: " + proxyUser); } diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java index c81888aff5eb043c43f3f6e99d3e35350b1e49ed..665266896f0f5fc793757974e2111f9e6dd25cbd 100644 --- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java +++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveServer2TransportMode; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hive.service.auth.HiveAuthFactory; @@ -66,6 +67,11 @@ public ThriftHttpCLIService(CLIService cliService, Runnable oomHook) { this.oomHook = oomHook; } + @Override + protected HiveServer2TransportMode getTransportMode() { + return HiveServer2TransportMode.http; + } + /** * Configure Jetty to serve http requests. Example of a client connection URL: * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target diff --git a/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistry.java b/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistry.java index 95c138861d5594269134b832e5823dd2e50451de..18cfa8d3111b8cdc5b4abd0f77f992736db4703c 100644 --- a/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistry.java +++ b/service/src/java/org/apache/hive/service/server/HS2ActivePassiveHARegistry.java @@ -346,7 +346,7 @@ void registerLeaderLatchListener(final LeaderLatchListener latchListener, final confsToPublish.put(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, conf.get(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname)); // Transport specific confs - if (HiveServer2.isHTTPTransportMode(conf)) { + if (HiveServer2.isHttpTransportMode(new HiveConf(conf, Configuration.class))) { confsToPublish.put(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT.varname, conf.get(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT.varname)); confsToPublish.put(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java index 181ea5d6d5c346381f946ad719243d8df3ecb0b9..3e7d127195a1a1715bb7fbab7691d991996ca549 100644 --- a/service/src/java/org/apache/hive/service/server/HiveServer2.java +++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -59,6 +59,7 @@ import org.apache.hadoop.hive.common.ZooKeeperHiveHelper; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveServer2TransportMode; import org.apache.hadoop.hive.llap.coordinator.LlapCoordinator; import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService; import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; @@ -220,12 +221,18 @@ public void run() { hiveServer2.stop(); } }; - if (isHTTPTransportMode(hiveConf)) { + + boolean isHttpTransportMode = isHttpTransportMode(hiveConf); + boolean isAllTransportMode = isAllTransportMode(hiveConf); + if (isHttpTransportMode || isAllTransportMode) { thriftCLIService = new ThriftHttpCLIService(cliService, oomHook); - } else { + addService(thriftCLIService); + } + if (!isHttpTransportMode || isAllTransportMode) { thriftCLIService = new ThriftBinaryCLIService(cliService, oomHook); + addService(thriftCLIService); //thriftCliService instance is used for zookeeper purposes } - addService(thriftCLIService); + super.init(hiveConf); // Set host name in conf try { @@ -448,12 +455,24 @@ private WMFullResourcePlan createTestResourcePlan() { return resourcePlan; } - public static boolean isHTTPTransportMode(Configuration hiveConf) { + public static boolean isHttpTransportMode(HiveConf hiveConf) { String transportMode = System.getenv("HIVE_SERVER2_TRANSPORT_MODE"); if (transportMode == null) { - transportMode = hiveConf.get(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname); + transportMode = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE); } - if (transportMode != null && (transportMode.equalsIgnoreCase("http"))) { + if (transportMode != null + && (transportMode.equalsIgnoreCase(HiveServer2TransportMode.http.toString()))) { + return true; + } + return false; + } + + public static boolean isAllTransportMode(HiveConf hiveConf) { + String transportMode = System.getenv("HIVE_SERVER2_TRANSPORT_MODE"); + if (transportMode == null) { + transportMode = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE); + } + if (transportMode != null && (transportMode.equalsIgnoreCase(HiveServer2TransportMode.all.toString()))) { return true; } return false; @@ -513,12 +532,16 @@ private void addConfsToPublish(HiveConf hiveConf, Map confsToPub confsToPublish.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, hiveConf.getVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE)); // Transport specific confs - if (isHTTPTransportMode(hiveConf)) { + boolean isHttpTransportMode = isHttpTransportMode(hiveConf); + boolean isAllTransportMode = isAllTransportMode(hiveConf); + + if (isHttpTransportMode || isAllTransportMode) { confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT.varname, Integer.toString(hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT))); confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH)); - } else { + } + if (!isHttpTransportMode || isAllTransportMode) { confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, Integer.toString(hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT))); confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_SASL_QOP.varname, diff --git a/service/src/test/org/apache/hive/service/cli/TestRetryingThriftCLIServiceClient.java b/service/src/test/org/apache/hive/service/cli/TestRetryingThriftCLIServiceClient.java index f3770c2d25499f2b028e83bf8f8a5324edd20945..cf4b5290515d6e729b9d59f3387c9064f6fc12df 100644 --- a/service/src/test/org/apache/hive/service/cli/TestRetryingThriftCLIServiceClient.java +++ b/service/src/test/org/apache/hive/service/cli/TestRetryingThriftCLIServiceClient.java @@ -19,6 +19,7 @@ package org.apache.hive.service.cli; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveServer2TransportMode; import org.apache.hive.service.Service; import org.apache.hive.service.auth.HiveAuthConstants; import org.apache.hive.service.cli.session.HiveSession; @@ -55,7 +56,7 @@ public void init() { hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, 15000); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION, HiveAuthConstants.AuthTypes.NONE.toString()); - hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE, "binary"); + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HiveServer2TransportMode.binary.toString()); hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_CLIENT_RETRY_LIMIT, 3); hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_CLIENT_CONNECTION_RETRY_LIMIT, 3); hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_ASYNC_EXEC_THREADS, 10);