diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 41d12ce643..7b7e3ecdef 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2301,7 +2301,10 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION("hive.server2.global.init.file.location", "${env:HIVE_CONF_DIR}", "Either the location of a HS2 global init file or a directory containing a .hiverc file. If the \n" + "property is set, the value must be a valid path to an init file or directory where the init file is located."), - HIVE_SERVER2_TRANSPORT_MODE("hive.server2.transport.mode", "binary", new StringSet("binary", "http"), + HIVE_SERVER2_TRANSPORT_MODE("hive.server2.transport.mode", + HiveServer2TransportMode.binary.toString(), + new StringSet(HiveServer2TransportMode.binary.toString(), + HiveServer2TransportMode.http.toString(), HiveServer2TransportMode.all.toString()), "Transport mode of HiveServer2."), HIVE_SERVER2_THRIFT_BIND_HOST("hive.server2.thrift.bind.host", "", "Bind host on which to run the HiveServer2 Thrift service."), diff --git common/src/java/org/apache/hadoop/hive/conf/HiveServer2TransportMode.java common/src/java/org/apache/hadoop/hive/conf/HiveServer2TransportMode.java new file mode 100644 index 0000000000..ed3cec016b --- /dev/null +++ common/src/java/org/apache/hadoop/hive/conf/HiveServer2TransportMode.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.conf; + +/** + * Hive Transport mode. + */ +public enum HiveServer2TransportMode { + /** + * Three modes: http, binary or all (which includes binary as well as http). + */ + http, binary, all +} diff --git itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcCookie.java itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcCookie.java index 5e70d68803..8ecc804c4a 100644 --- itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcCookie.java +++ itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcCookie.java @@ -21,7 +21,10 @@ import java.io.File; import java.io.IOException; import java.sql.Connection; +import java.sql.DriverManager; import java.sql.Statement; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.Properties; import java.util.concurrent.TimeUnit; @@ -36,49 +39,69 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +@RunWith(Parameterized.class) public class TestJdbcWithMiniKdcCookie { private static MiniHS2 miniHS2 = null; - private static MiniHiveKdc miniHiveKdc = null; - private Connection hs2Conn; + private MiniHiveKdc miniHiveKdc = null; + private static Connection hs2Conn; File dataFile; - protected static HiveConf hiveConf = new HiveConf(); + protected static HiveConf hiveConf = null; private static String HIVE_NON_EXISTENT_USER = "hive_no_exist"; + @Parameterized.Parameter + public String transportMode = null; + + @Parameterized.Parameters(name = "{index}: tranportMode={0}") + public static Collection transportModes() { + return Arrays.asList(new Object[][] { + { MiniHS2.HS2_ALL_MODE },{ MiniHS2.HS2_HTTP_MODE} + }); + } + + @BeforeClass public static void beforeTest() throws Exception { - hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, MiniHS2.HS2_HTTP_MODE); + Class.forName(MiniHS2.getJdbcDriverName()); + } + + @Before + public void setUp() throws Exception { + DriverManager.setLoginTimeout(0); + hiveConf = new HiveConf(); + hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, transportMode); System.err.println("Testing using HS2 mode : " - + hiveConf.getVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE)); + + hiveConf.getVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE)); hiveConf.setBoolVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_AUTH_ENABLED, - true); + true); // set a small time unit as cookie max age so that the server sends a 401 hiveConf.setTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_MAX_AGE, - 1, TimeUnit.SECONDS); + 1, TimeUnit.SECONDS); hiveConf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); - miniHiveKdc = MiniHiveKdc.getMiniHiveKdc(hiveConf); - miniHS2 = MiniHiveKdc.getMiniHS2WithKerb(miniHiveKdc, hiveConf); - miniHS2.start(new HashMap()); + startMiniHS2(); } - - @Before - public void setUp() throws Exception { + private void startMiniHS2() { + try { + miniHiveKdc = MiniHiveKdc.getMiniHiveKdc(hiveConf); + miniHS2 = MiniHiveKdc.getMiniHS2WithKerb(miniHiveKdc, hiveConf); + miniHS2.start(new HashMap()); + } catch (Exception e) { + e.printStackTrace(); + Assert.assertNotNull(e.getMessage()); + } } - @After public void tearDown() throws Exception { if (hs2Conn != null) { - try { - hs2Conn.close(); - } catch (Exception e) { - // Ignore shutdown errors since there are negative tests - } + hs2Conn.close(); + hs2Conn = null; + } + if (miniHS2 != null && miniHS2.isStarted()) { + miniHS2.stop(); + miniHS2.cleanup(); } - } - - @AfterClass - public static void afterTest() throws Exception { - miniHS2.stop(); } @Test @@ -99,9 +122,10 @@ public void testCookie() throws Exception { } stmt.execute("drop table " + tableName); stmt.close(); + + testCookieNegative(); } - @Test public void testCookieNegative() throws Exception { try { // Trying to connect with a non-existent user should still fail with @@ -114,6 +138,6 @@ public void testCookieNegative() throws Exception { private Connection getConnection(String userName) throws Exception { miniHiveKdc.loginUser(userName); - return new HiveConnection(miniHS2.getJdbcURL(), new Properties()); + return new HiveConnection(miniHS2.getHttpJdbcURL(), new Properties()); } } diff --git itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcSQLAuthAll.java itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcSQLAuthAll.java new file mode 100644 index 0000000000..bf778ccc21 --- /dev/null +++ itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdcSQLAuthAll.java @@ -0,0 +1,14 @@ +package org.apache.hive.minikdc; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.junit.BeforeClass; + +public class TestJdbcWithMiniKdcSQLAuthAll extends JdbcWithMiniKdcSQLAuthTest { + + @BeforeClass + public static void beforeTest() throws Exception { + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE, MiniHS2.HS2_ALL_MODE); + JdbcWithMiniKdcSQLAuthTest.beforeTestBase(); + } +} diff --git itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/BeelineWithHS2ConnectionFileTestBase.java itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/BeelineWithHS2ConnectionFileTestBase.java index 2fb64536a6..a2ff085eeb 100644 --- itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/BeelineWithHS2ConnectionFileTestBase.java +++ itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/BeelineWithHS2ConnectionFileTestBase.java @@ -46,7 +46,10 @@ import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +@RunWith(Parameterized.class) public abstract class BeelineWithHS2ConnectionFileTestBase { protected MiniHS2 miniHS2; protected HiveConf hiveConf = new HiveConf(); @@ -64,6 +67,10 @@ protected Map confOverlay = new HashMap<>(); + @Parameterized.Parameter + public String transportMode = null; + + protected class TestBeeLine extends BeeLine { UserHS2ConnectionFileParser testHs2ConfigFileManager; ByteArrayOutputStream os; @@ -162,7 +169,7 @@ public void before() throws Exception { miniHS2 = getNewMiniHS2(); confOverlay = new HashMap(); confOverlay.put(ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); - confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, "binary"); + confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, transportMode); } protected MiniHS2 getNewMiniHS2() throws Exception { diff --git itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java index 8bff87c0bf..76396f5f86 100644 --- itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java +++ itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineConnectionUsingHiveSite.java @@ -18,20 +18,43 @@ package org.apache.hive.beeline.hs2connection; import java.io.File; +import java.util.Arrays; +import java.util.Collection; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.junit.Assume; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +@RunWith(Parameterized.class) public class TestBeelineConnectionUsingHiveSite extends BeelineWithHS2ConnectionFileTestBase { + + boolean isHttpModeTest = false; + + @Parameterized.Parameters(name = "{index}: tranportMode={0}") + public static Collection transportModes() { + return Arrays.asList(new Object[][] { + { MiniHS2.HS2_BINARY_MODE},{ MiniHS2.HS2_HTTP_MODE}, + { MiniHS2.HS2_ALL_MODE } + }); + } @Test public void testBeelineConnectionHttp() throws Exception { + Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_HTTP_MODE) + || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE)); + isHttpModeTest = true; setupHs2(); String path = createDefaultHs2ConnectionFile(); testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); + isHttpModeTest = false; } @Test public void testBeelineConnectionSSL() throws Exception { + Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_BINARY_MODE) + || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE)); setupSSLHs2(); String path = createDefaultHs2ConnectionFile(); testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); @@ -39,6 +62,8 @@ public void testBeelineConnectionSSL() throws Exception { @Test public void testBeelineConnectionNoAuth() throws Exception { + Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_BINARY_MODE) + || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE)); setupNoAuthHs2(); String path = createDefaultHs2ConnectionFile(); testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); @@ -50,12 +75,16 @@ public void testBeelineConnectionNoAuth() throws Exception { */ @Test public void testBeelineWithNoConnectionFile() throws Exception { + Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_BINARY_MODE) + || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE)); setupNoAuthHs2(); testBeeLineConnection(null, new String[] { "-e", "show tables;" }, "no current connection"); } @Test public void testBeelineUsingArgs() throws Exception { + Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_BINARY_MODE) + || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE)); setupNoAuthHs2(); String url = miniHS2.getBaseJdbcURL() + "default"; String args[] = new String[] { "-u", url, "-n", System.getProperty("user.name"), "-p", "foo", @@ -82,8 +111,6 @@ private void setupSSLHs2() throws Exception { } private void setupHs2() throws Exception { - confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_HTTP_MODE); - confOverlay.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, HS2_HTTP_ENDPOINT); confOverlay.put(ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true"); miniHS2.start(confOverlay); createTable(); @@ -92,6 +119,8 @@ private void setupHs2() throws Exception { private String createDefaultHs2ConnectionFile() throws Exception { Hs2ConnectionXmlConfigFileWriter writer = new Hs2ConnectionXmlConfigFileWriter(); String baseJdbcURL = miniHS2.getBaseJdbcURL(); + if(isHttpModeTest) + baseJdbcURL = miniHS2.getBaseHttpJdbcURL(); System.out.println(baseJdbcURL); writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "user", System.getProperty("user.name")); diff --git itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithUserHs2ConnectionFile.java itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithUserHs2ConnectionFile.java index 3684f14db5..8389c951ed 100644 --- itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithUserHs2ConnectionFile.java +++ itests/hive-unit/src/test/java/org/apache/hive/beeline/hs2connection/TestBeelineWithUserHs2ConnectionFile.java @@ -19,22 +19,36 @@ import java.io.File; import java.net.URI; +import java.util.Arrays; +import java.util.Collection; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.junit.Assume; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +@RunWith(Parameterized.class) public class TestBeelineWithUserHs2ConnectionFile extends BeelineWithHS2ConnectionFileTestBase { + @Parameterized.Parameters(name = "{index}: tranportMode={0}") + public static Collection transportModes() { + return Arrays.asList(new Object[][] { + { MiniHS2.HS2_ALL_MODE },{ MiniHS2.HS2_BINARY_MODE},{ MiniHS2.HS2_HTTP_MODE} + }); + } + @Test public void testBeelineConnectionHttp() throws Exception { + Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_HTTP_MODE) + || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE)); setupHttpHs2(); String path = createHttpHs2ConnectionFile(); testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); } private void setupHttpHs2() throws Exception { - confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, HS2_HTTP_MODE); - confOverlay.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, HS2_HTTP_ENDPOINT); confOverlay.put(ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "true"); miniHS2.start(confOverlay); createTable(); @@ -42,7 +56,7 @@ private void setupHttpHs2() throws Exception { private String createHttpHs2ConnectionFile() throws Exception { Hs2ConnectionXmlConfigFileWriter writer = new Hs2ConnectionXmlConfigFileWriter(); - String baseJdbcURL = miniHS2.getBaseJdbcURL(); + String baseJdbcURL = miniHS2.getBaseHttpJdbcURL(); URI uri = new URI(baseJdbcURL.substring(5)); writer.writeProperty(HS2ConnectionFileParser.BEELINE_CONNECTION_PROPERTY_PREFIX + "hosts", @@ -62,6 +76,8 @@ private String createHttpHs2ConnectionFile() throws Exception { @Test public void testBeelineConnectionNoAuth() throws Exception { + Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_BINARY_MODE) + || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE)); setupNoAuthConfHS2(); String path = createNoAuthHs2ConnectionFile(); testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); @@ -89,6 +105,8 @@ private String createNoAuthHs2ConnectionFile() throws Exception { @Test public void testBeelineConnectionSSL() throws Exception { + Assume.assumeTrue(transportMode.equals(MiniHS2.HS2_BINARY_MODE) + || transportMode.equalsIgnoreCase(MiniHS2.HS2_ALL_MODE)); setupSslHs2(); String path = createSSLHs2ConnectionFile(); testBeeLineConnection(path, new String[] { "-e", "show tables;" }, tableName); diff --git itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestXSRFFilter.java itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestXSRFFilter.java index 88a403a0d5..e80a7b319f 100644 --- itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestXSRFFilter.java +++ itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestXSRFFilter.java @@ -26,6 +26,8 @@ import java.sql.SQLException; import java.sql.Statement; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -38,10 +40,14 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +@RunWith(Parameterized.class) public class TestXSRFFilter { private static MiniHS2 miniHS2 = null; @@ -51,6 +57,17 @@ private Connection hs2Conn = null; + @Parameterized.Parameter + public String transportMode = null; + + @Parameterized.Parameters(name = "{index}: tranportMode={0}") + public static Collection transportModes() { + return Arrays.asList(new Object[][] { + { MiniHS2.HS2_ALL_MODE },{ MiniHS2.HS2_HTTP_MODE} + }); + } + + @BeforeClass public static void beforeClass() throws IOException { MiniHS2.cleanupLocalDir(); @@ -72,7 +89,7 @@ private void initHS2(boolean enableXSRFFilter) throws Exception { kvDataFilePath = new Path(dataFileDir, "kv1.txt"); Map confOverlay = new HashMap(); confOverlay.put(ConfVars.HIVE_SERVER2_XSRF_FILTER_ENABLED.varname, String.valueOf(enableXSRFFilter)); - confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, "http"); + confOverlay.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, transportMode); miniHS2.start(confOverlay); } @@ -138,7 +155,7 @@ private void runTest(boolean filterEnabled, boolean injectionEnabled) throws Exc private void runBasicCommands() throws Exception { - hs2Conn = getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar"); + hs2Conn = getConnection(miniHS2.getHttpJdbcURL(), System.getProperty("user.name"), "bar"); String tableName = "testTab1"; Statement stmt = hs2Conn.createStatement(); diff --git itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithAllandBinary.java itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithAllandBinary.java new file mode 100644 index 0000000000..70dce8da36 --- /dev/null +++ itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithAllandBinary.java @@ -0,0 +1,61 @@ +package org.apache.hive.service.cli.thrift; + +import static org.junit.Assert.assertNotNull; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.Service; +import org.apache.hive.service.auth.HiveAuthFactory; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +/** + * TestThriftHttpCLIService. + * This tests ThriftCLIService started in http mode. + */ + +public class TestThriftCLIServiceWithAllandBinary extends ThriftCLIServiceTest { + + private static String transportMode = "all"; + + /** + * @throws Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Set up the base class + ThriftCLIServiceTest.setUpBeforeClass(); + + assertNotNull(port); + assertNotNull(hiveServer2); + assertNotNull(hiveConf); + + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, host); + hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, port); + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION, HiveAuthFactory.AuthTypes.NONE.toString()); + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE, transportMode); + + startHiveServer2WithConf(hiveConf); + + client = getHttpServiceClientInternal(); + } + + /** + * @throws Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + ThriftCLIServiceTest.tearDownAfterClass(); + } + static ThriftCLIServiceClient getHttpServiceClientInternal() { + for (Service service : hiveServer2.getServices()) { + if (service instanceof ThriftBinaryCLIService) { + return new ThriftCLIServiceClient((ThriftBinaryCLIService) service); + } + if (service instanceof ThriftHttpCLIService) { + continue; + } + } + throw new IllegalStateException("HiveServer2 not running Thrift service"); + } +} diff --git itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithAllandHttp.java itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithAllandHttp.java new file mode 100644 index 0000000000..dbd9a13c79 --- /dev/null +++ itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/TestThriftCLIServiceWithAllandHttp.java @@ -0,0 +1,63 @@ +package org.apache.hive.service.cli.thrift; + +import static org.junit.Assert.assertNotNull; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hive.service.Service; +import org.apache.hive.service.auth.HiveAuthFactory; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +/** + * TestThriftHttpCLIService. + * This tests ThriftCLIService started in http mode. + */ + +public class TestThriftCLIServiceWithAllandHttp extends ThriftCLIServiceTest { + + private static String transportMode = "all"; + private static String thriftHttpPath = "cliservice"; + + /** + * @throws Exception + */ + @BeforeClass + public static void setUpBeforeClass() throws Exception { + // Set up the base class + ThriftCLIServiceTest.setUpBeforeClass(); + + assertNotNull(port); + assertNotNull(hiveServer2); + assertNotNull(hiveConf); + + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, host); + hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT, port); + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION, HiveAuthFactory.AuthTypes.NOSASL.toString()); + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE, transportMode); + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH, thriftHttpPath); + + startHiveServer2WithConf(hiveConf); + + client = getHttpServiceClientInternal(); + } + + /** + * @throws Exception + */ + @AfterClass + public static void tearDownAfterClass() throws Exception { + ThriftCLIServiceTest.tearDownAfterClass(); + } + static ThriftCLIServiceClient getHttpServiceClientInternal() { + for (Service service : hiveServer2.getServices()) { + if (service instanceof ThriftBinaryCLIService) { + continue; + } + if (service instanceof ThriftHttpCLIService) { + return new ThriftCLIServiceClient((ThriftHttpCLIService) service); + } + } + throw new IllegalStateException("HiveServer2 not running Thrift service"); + } +} diff --git itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index 71f9640ad2..8f265b02a2 100644 --- itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -57,6 +57,7 @@ public static final String HS2_BINARY_MODE = "binary"; public static final String HS2_HTTP_MODE = "http"; + public static final String HS2_ALL_MODE = "all"; private static final String driverName = "org.apache.hive.jdbc.HiveDriver"; private static final FsPermission FULL_PERM = new FsPermission((short)00777); private static final FsPermission WRITE_ALL_PERM = new FsPermission((short)00733); @@ -135,6 +136,7 @@ public Builder withHA() { */ public Builder withHTTPTransport(){ this.isHTTPTransMode = true; + hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_HTTP_MODE); return this; } @@ -147,11 +149,6 @@ public MiniHS2 build() throws Exception { if (miniClusterType == MiniClusterType.MR && useMiniKdc) { throw new IOException("Can't create secure miniMr ... yet"); } - if (isHTTPTransMode) { - hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_HTTP_MODE); - } else { - hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_BINARY_MODE); - } return new MiniHS2(hiveConf, miniClusterType, useMiniKdc, serverPrincipal, serverKeytab, isMetastoreRemote, usePortsFromConf, authType, isHA, cleanupLocalDirOnStartup); } @@ -466,6 +463,18 @@ public String getBaseJdbcURL() { } } + /** + * Build base JDBC URL + * @return + */ + public String getBaseHttpJdbcURL() { + String transportMode = getConfProperty(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname); + if(!transportMode.equalsIgnoreCase(HS2_ALL_MODE)) { + return getBaseJdbcURL(); + } + return "jdbc:hive2://" + getHost() + ":" + getBinaryPort() + "/"; + } + /** * Build zk base JDBC URL * @return @@ -479,6 +488,23 @@ private String getZKBaseJdbcURL() throws Exception { throw new Exception("Server's HiveConf is null. Unable to read ZooKeeper configs."); } + /** + * return HTTP connection URL for this server instance + * @return + * @throws Exception + */ + public String getHttpJdbcURL() throws Exception { + String transportMode = getConfProperty(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname); + if(!transportMode.equalsIgnoreCase(HS2_ALL_MODE)) { + return getJdbcURL(); + } + //CAUTION: This will result in concurrency issues. + getHiveConf().setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_HTTP_MODE); + String url = getJdbcURL("default"); + getHiveConf().setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_ALL_MODE); + return url; + } + private boolean isHttpTransportMode() { String transportMode = getConfProperty(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname); return transportMode != null && (transportMode.equalsIgnoreCase(HS2_HTTP_MODE)); diff --git service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java index 0fdc8d9074..ae1a28627e 100644 --- service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java +++ service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.common.log.ProgressMonitor; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveServer2TransportMode; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.shims.HadoopShims.KerberosNameShim; import org.apache.hadoop.hive.shims.ShimLoader; @@ -178,7 +179,7 @@ public synchronized void init(HiveConf hiveConf) { // Initialize common server configs needed in both binary & http modes String portString; // HTTP mode - if (HiveServer2.isHTTPTransportMode(hiveConf)) { + if (getTransportMode() == HiveServer2TransportMode.http) { workerKeepAliveTime = hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_WORKER_KEEPALIVE_TIME, TimeUnit.SECONDS); @@ -347,8 +348,10 @@ private String getIpAddress() { String clientIpAddress; // Http transport mode. // We set the thread local ip address, in ThriftHttpServlet. - if (cliService.getHiveConf().getVar( - ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) { + //Since we are allowing multiple transport modes, this + //transport information comes from the subclasses + //instead of reading from hive configuration + if (getTransportMode() == HiveServer2TransportMode.http) { clientIpAddress = SessionManager.getIpAddress(); } else { @@ -386,8 +389,7 @@ private String getUserName(TOpenSessionReq req) throws HiveSQLException, IOExcep } // Http transport mode. // We set the thread local username, in ThriftHttpServlet. - if (cliService.getHiveConf().getVar( - ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) { + if (getTransportMode() == HiveServer2TransportMode.http) { userName = SessionManager.getUserName(); } if (userName == null) { @@ -801,6 +803,14 @@ public TGetCrossReferenceResp GetCrossReference(TGetCrossReferenceReq req) @Override public abstract void run(); + /** Transport mode of hiveserver2 thrift + * Default is binary transport mode. + * @return + */ + protected HiveServer2TransportMode getTransportMode() { + return HiveServer2TransportMode.binary; + } + /** * If the proxy user name is provided then check privileges to substitute the user. * @param realUser @@ -814,8 +824,7 @@ private String getProxyUser(String realUser, Map sessionConf, String proxyUser = null; // Http transport mode. // We set the thread local proxy username, in ThriftHttpServlet. - if (cliService.getHiveConf().getVar( - ConfVars.HIVE_SERVER2_TRANSPORT_MODE).equalsIgnoreCase("http")) { + if (getTransportMode() == HiveServer2TransportMode.http) { proxyUser = SessionManager.getProxyUserName(); LOG.debug("Proxy user from query string: " + proxyUser); } diff --git service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java index ebec165fed..59c3b59703 100644 --- service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java +++ service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveServer2TransportMode; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.security.UserGroupInformation; @@ -55,6 +56,11 @@ public ThriftHttpCLIService(CLIService cliService, Runnable oomHook) { this.oomHook = oomHook; } + @Override + protected HiveServer2TransportMode getTransportMode() { + return HiveServer2TransportMode.http; + } + /** * Configure Jetty to serve http requests. Example of a client connection URL: * http://localhost:10000/servlets/thrifths2/ A gateway may cause actual target URL to differ, diff --git service/src/java/org/apache/hive/service/server/HiveServer2.java service/src/java/org/apache/hive/service/server/HiveServer2.java index e5f449122b..3a626d97c6 100644 --- service/src/java/org/apache/hive/service/server/HiveServer2.java +++ service/src/java/org/apache/hive/service/server/HiveServer2.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveServer2TransportMode; import org.apache.hadoop.hive.llap.coordinator.LlapCoordinator; import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService; import org.apache.hadoop.hive.ql.exec.spark.session.SparkSessionManagerImpl; @@ -91,7 +92,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; /** * HiveServer2. @@ -133,12 +133,18 @@ public void run() { hiveServer2.stop(); } }; - if (isHTTPTransportMode(hiveConf)) { + + boolean isHttpTransportMode = isHttpTransportMode(hiveConf); + boolean isAllTransportMode = isAllTransportMode(hiveConf); + if (isHttpTransportMode || isAllTransportMode) { thriftCLIService = new ThriftHttpCLIService(cliService, oomHook); - } else { + addService(thriftCLIService); + } + if (!isHttpTransportMode || isAllTransportMode) { thriftCLIService = new ThriftBinaryCLIService(cliService, oomHook); + addService(thriftCLIService); //thriftCliService instance is used for zookeeper purposes } - addService(thriftCLIService); + super.init(hiveConf); // Set host name in hiveConf try { @@ -240,12 +246,25 @@ public void run() { }); } - public static boolean isHTTPTransportMode(HiveConf hiveConf) { + public static boolean isHttpTransportMode(HiveConf hiveConf) { + String transportMode = System.getenv("HIVE_SERVER2_TRANSPORT_MODE"); + if (transportMode == null) { + transportMode = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE); + } + if (transportMode != null + && (transportMode.equalsIgnoreCase(HiveServer2TransportMode.http.toString()))) { + return true; + } + return false; + } + + public static boolean isAllTransportMode(HiveConf hiveConf) { String transportMode = System.getenv("HIVE_SERVER2_TRANSPORT_MODE"); if (transportMode == null) { transportMode = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE); } - if (transportMode != null && (transportMode.equalsIgnoreCase("http"))) { + if (transportMode != null + && (transportMode.equalsIgnoreCase(HiveServer2TransportMode.all.toString()))) { return true; } return false; @@ -379,12 +398,16 @@ private void addConfsToPublish(HiveConf hiveConf, Map confsToPub confsToPublish.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname, hiveConf.getVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE)); // Transport specific confs - if (isHTTPTransportMode(hiveConf)) { + boolean isHttpTransportMode = isHttpTransportMode(hiveConf); + boolean isAllTransportMode = isAllTransportMode(hiveConf); + + if (isHttpTransportMode || isAllTransportMode) { confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT.varname, Integer.toString(hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT))); confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname, hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH)); - } else { + } + if (!isHttpTransportMode || isAllTransportMode) { confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, Integer.toString(hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT))); confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_SASL_QOP.varname, diff --git service/src/test/org/apache/hive/service/cli/TestRetryingThriftCLIServiceClient.java service/src/test/org/apache/hive/service/cli/TestRetryingThriftCLIServiceClient.java index 79953c4842..205ec8c9e3 100644 --- service/src/test/org/apache/hive/service/cli/TestRetryingThriftCLIServiceClient.java +++ service/src/test/org/apache/hive/service/cli/TestRetryingThriftCLIServiceClient.java @@ -19,6 +19,7 @@ package org.apache.hive.service.cli; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveServer2TransportMode; import org.apache.hive.service.Service; import org.apache.hive.service.auth.HiveAuthFactory; import org.apache.hive.service.cli.session.HiveSession; @@ -55,7 +56,7 @@ public void init() { hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT, 15000); hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS, false); hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION, HiveAuthFactory.AuthTypes.NONE.toString()); - hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE, "binary"); + hiveConf.setVar(HiveConf.ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HiveServer2TransportMode.binary.toString()); hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_CLIENT_RETRY_LIMIT, 3); hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_CLIENT_CONNECTION_RETRY_LIMIT, 3); hiveConf.setIntVar(HiveConf.ConfVars.HIVE_SERVER2_ASYNC_EXEC_THREADS, 10); diff --git service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java index 6fec947d62..2572c7df1e 100644 --- service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java +++ service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.conf.HiveServer2TransportMode; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hive.service.Service; import org.apache.hive.service.auth.HiveAuthFactory.AuthTypes; @@ -69,7 +70,7 @@ public static void setUpBeforeClass() throws Exception { // Set the cookie max age to a very low value so that // the server sends 401 very frequently hiveConf.setTimeVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_MAX_AGE, 1, TimeUnit.SECONDS); - hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, "http"); + hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HiveServer2TransportMode.http.toString()); hiveConf.setVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH, "cliservice"); assertNotNull(port);