diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 7f4afd9..ad375db 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -204,11 +204,15 @@ PLAN_SERIALIZATION("hive.plan.serialization.format", "kryo", "Query plan format serialization between client and task nodes. \n" + "Two supported values are : kryo and javaXML. Kryo is default."), - SCRATCHDIR("hive.exec.scratchdir", "/tmp/hive-${system:user.name}", "Scratch space for Hive jobs"), + SCRATCHDIR_PREFIX("hive.exec.scratchdir.prefix", "/tmp/hive-", "HDFS scratch dir prefix"), + SCRATCHDIR("hive.exec.scratchdir", "${hive.exec.scratchdir.prefix}${system:user.name}", "HDFS scratch dir for Hive jobs"), LOCALSCRATCHDIR("hive.exec.local.scratchdir", "${system:java.io.tmpdir}" + File.separator + "${system:user.name}", "Local scratch space for Hive jobs"), SCRATCHDIRPERMISSION("hive.scratch.dir.permission", "700", ""), + DOWNLOADED_RESOURCES_DIR("hive.downloaded.resources.dir", + "${system:java.io.tmpdir}" + File.separator + "${hive.session.id}_resources", + "Temporary local directory for added resources in the remote file system."), SUBMITVIACHILD("hive.exec.submitviachild", false, ""), SUBMITLOCALTASKVIACHILD("hive.exec.submit.local.task.via.child", true, "Determines whether local tasks (typically mapjoin hashtable generation phase) runs in \n" + @@ -272,9 +276,6 @@ "Maximum number of dynamic partitions allowed to be created in each mapper/reducer node."), MAXCREATEDFILES("hive.exec.max.created.files", 100000L, "Maximum number of HDFS files created by all mappers/reducers in a MapReduce job."), - DOWNLOADED_RESOURCES_DIR("hive.downloaded.resources.dir", - "${system:java.io.tmpdir}" + File.separator + "${hive.session.id}_resources", - "Temporary local directory for added resources in the remote file system."), DEFAULTPARTITIONNAME("hive.exec.default.partition.name", "__HIVE_DEFAULT_PARTITION__", "The default partition name in case the dynamic partition column value is null/empty string or any other values that cannot be escaped. \n" + "This value must not contain any special character used in HDFS URI (e.g., ':', '%', '/' etc). \n" + diff --git a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index 4502439..2adbcb6 100644 --- a/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ b/itests/hive-unit/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -30,7 +30,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.metastore.HiveMetaStore; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim; import org.apache.hadoop.hive.shims.HadoopShims.MiniMrShim; @@ -55,6 +54,7 @@ private static final AtomicLong hs2Counter = new AtomicLong(); private MiniMrShim mr; private MiniDFSShim dfs; + private FileSystem localFS; private boolean useMiniMR = false; private boolean useMiniKdc = false; private final String serverPrincipal; @@ -127,6 +127,10 @@ public void setDfs(MiniDFSShim dfs) { this.dfs = dfs; } + public FileSystem getLocalFS() { + return localFS; + } + public boolean isUseMiniMR() { return useMiniMR; } @@ -145,7 +149,8 @@ private MiniHS2(HiveConf hiveConf, boolean useMiniMR, boolean useMiniKdc, String this.useMiniKdc = useMiniKdc; this.serverPrincipal = serverPrincipal; this.serverKeytab = serverKeytab; - baseDir = Files.createTempDir(); + baseDir = Files.createTempDir(); + localFS = FileSystem.getLocal(hiveConf); FileSystem fs; if (useMiniMR) { dfs = ShimLoader.getHadoopShims().getMiniDfs(hiveConf, 4, true, null); @@ -341,7 +346,7 @@ private void waitForStartup() throws Exception { getMiniKdc().loginUser(getMiniKdc().getDefaultUserPrincipal()); sessionConf.put("principal", serverPrincipal); } - */ + */ sessionHandle = hs2Client.openSession("foo", "bar", sessionConf); } catch (Exception e) { // service not started yet diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestUtilitiesDfs.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestUtilitiesDfs.java deleted file mode 100644 index 69d88c1..0000000 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestUtilitiesDfs.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.ql; - -import java.io.IOException; - -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.ql.exec.Utilities; -import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim; -import org.apache.hadoop.hive.shims.ShimLoader; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - -public class TestUtilitiesDfs { - private static final FsPermission FULL_PERM = new FsPermission((short) 00777); - private static MiniDFSShim dfs; - private static HiveConf conf; - - @BeforeClass - public static void setupDfs() throws Exception { - conf = new HiveConf(); - dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null); - - assertNotNull("MiniDFS is not initialized", dfs); - assertNotNull("HiveConf is not initialized", conf); - } - - @Test - public void testCreateDirWithPermissionRecursive() throws IllegalArgumentException, IOException { - FileSystem fs = dfs.getFileSystem(); - Path dir = new Path(new Path(fs.getUri()), "/testUtilitiesUMaskReset"); - Utilities.createDirsWithPermission(conf, dir, FULL_PERM, true); - FileStatus status = fs.getFileStatus(dir); - assertEquals("Created dir has invalid permissions.", - FULL_PERM.toString(), status.getPermission().toString()); - } - - @AfterClass - public static void shutdownDfs() throws Exception { - if (dfs != null) { - dfs.shutdown(); - dfs = null; - } - } -} diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java index 0e769ae..9c42161 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcWithMiniHS2.java @@ -31,7 +31,9 @@ import java.util.HashMap; import java.util.Map; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hive.jdbc.miniHS2.MiniHS2; @@ -41,255 +43,343 @@ import org.junit.BeforeClass; import org.junit.Test; - public class TestJdbcWithMiniHS2 { - private static MiniHS2 miniHS2 = null; - private static Path dataFilePath; - - private Connection hs2Conn = null; - - @BeforeClass - public static void beforeTest() throws Exception { - Class.forName(MiniHS2.getJdbcDriverName()); - HiveConf conf = new HiveConf(); - conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); - miniHS2 = new MiniHS2(conf); - String dataFileDir = conf.get("test.data.files").replace('\\', '/') - .replace("c:", ""); - dataFilePath = new Path(dataFileDir, "kv1.txt"); - Map confOverlay = new HashMap(); - miniHS2.start(confOverlay); - } +public class TestJdbcWithMiniHS2 { + private static MiniHS2 miniHS2 = null; + private static Path dataFilePath; + + private Connection hs2Conn = null; + + @BeforeClass + public static void beforeTest() throws Exception { + Class.forName(MiniHS2.getJdbcDriverName()); + HiveConf conf = new HiveConf(); + conf.setBoolVar(ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + miniHS2 = new MiniHS2(conf); + String dataFileDir = conf.get("test.data.files").replace('\\', '/') + .replace("c:", ""); + dataFilePath = new Path(dataFileDir, "kv1.txt"); + Map confOverlay = new HashMap(); + miniHS2.start(confOverlay); + } - @Before - public void setUp() throws Exception { - hs2Conn = getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar"); - } + @Before + public void setUp() throws Exception { + hs2Conn = getConnection(miniHS2.getJdbcURL(), System.getProperty("user.name"), "bar"); + } - private Connection getConnection(String jdbcURL, String user, String pwd) throws SQLException { - Connection conn = DriverManager.getConnection(jdbcURL, user, pwd); - conn.createStatement().execute("set hive.support.concurrency = false"); - return conn; - } + private Connection getConnection(String jdbcURL, String user, String pwd) throws SQLException { + Connection conn = DriverManager.getConnection(jdbcURL, user, pwd); + conn.createStatement().execute("set hive.support.concurrency = false"); + return conn; + } + + @After + public void tearDown() throws Exception { + hs2Conn.close(); + } - @After - public void tearDown() throws Exception { - hs2Conn.close(); + @AfterClass + public static void afterTest() throws Exception { + if (miniHS2.isStarted()) { + miniHS2.stop(); } + } - @AfterClass - public static void afterTest() throws Exception { - if (miniHS2.isStarted()) - miniHS2.stop(); + @Test + public void testConnection() throws Exception { + String tableName = "testTab1"; + Statement stmt = hs2Conn.createStatement(); + + // create table + stmt.execute("DROP TABLE IF EXISTS " + tableName); + stmt.execute("CREATE TABLE " + tableName + + " (under_col INT COMMENT 'the under column', value STRING) COMMENT ' test table'"); + + // load data + stmt.execute("load data local inpath '" + + dataFilePath.toString() + "' into table " + tableName); + + ResultSet res = stmt.executeQuery("SELECT * FROM " + tableName); + assertTrue(res.next()); + assertEquals("val_238", res.getString(2)); + res.close(); + stmt.close(); + } + + + /** This test is to connect to any database without using the command "Use <>" + * 1)connect to default database. + * 2) Create a new DB test_default. + * 3) Connect to test_default database. + * 4) Connect and create table under test_default_test. + * 5) Connect and display all tables. + * 6) Connect to default database and shouldn't find table test_default_test. + * 7) Connect and drop test_default_test. + * 8) drop test_default database. + */ + + @Test + public void testURIDatabaseName() throws Exception{ + + String jdbcUri = miniHS2.getJdbcURL().substring(0, miniHS2.getJdbcURL().indexOf("default")); + + hs2Conn= getConnection(jdbcUri+"default", System.getProperty("user.name"),"bar"); + String dbName="test_connection_non_default_db"; + String tableInNonDefaultSchema="table_in_non_default_schema"; + Statement stmt = hs2Conn.createStatement(); + stmt.execute("create database if not exists "+dbName); + stmt.close(); + hs2Conn.close(); + + hs2Conn = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar"); + stmt = hs2Conn .createStatement(); + boolean expected = stmt.execute(" create table "+tableInNonDefaultSchema +" (x int)"); + stmt.close(); + hs2Conn .close(); + + hs2Conn = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar"); + stmt = hs2Conn .createStatement(); + ResultSet res = stmt.executeQuery("show tables"); + boolean testTableExists = false; + while (res.next()) { + assertNotNull("table name is null in result set", res.getString(1)); + if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) { + testTableExists = true; + } + } + assertTrue("table name "+tableInNonDefaultSchema + + " found in SHOW TABLES result set", testTableExists); + stmt.close(); + hs2Conn .close(); + + hs2Conn = getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar"); + stmt = hs2Conn .createStatement(); + res = stmt.executeQuery("show tables"); + testTableExists = false; + while (res.next()) { + assertNotNull("table name is null in result set", res.getString(1)); + if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) { + testTableExists = true; + } } - @Test - public void testConnection() throws Exception { - String tableName = "testTab1"; - Statement stmt = hs2Conn.createStatement(); - - // create table - stmt.execute("DROP TABLE IF EXISTS " + tableName); - stmt.execute("CREATE TABLE " + tableName - + " (under_col INT COMMENT 'the under column', value STRING) COMMENT ' test table'"); - - // load data - stmt.execute("load data local inpath '" - + dataFilePath.toString() + "' into table " + tableName); - - ResultSet res = stmt.executeQuery("SELECT * FROM " + tableName); - assertTrue(res.next()); - assertEquals("val_238", res.getString(2)); - res.close(); - stmt.close(); + assertFalse("table name "+tableInNonDefaultSchema + + " NOT found in SHOW TABLES result set", testTableExists); + stmt.close(); + hs2Conn .close(); + + hs2Conn = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar"); + stmt = hs2Conn .createStatement(); + stmt.execute("set hive.support.concurrency = false"); + res = stmt.executeQuery("show tables"); + + stmt.execute(" drop table if exists table_in_non_default_schema"); + expected = stmt.execute("DROP DATABASE "+ dbName); + stmt.close(); + + hs2Conn = getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar"); + stmt = hs2Conn .createStatement(); + res = stmt.executeQuery("show tables"); + testTableExists = false; + while (res.next()) { + assertNotNull("table name is null in result set", res.getString(1)); + if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) { + testTableExists = true; + } } + // test URI with no dbName + hs2Conn = getConnection(jdbcUri, System.getProperty("user.name"),"bar"); + verifyCurrentDB("default", hs2Conn); + hs2Conn.close(); - /** This test is to connect to any database without using the command "Use <>" - * 1)connect to default database. - * 2) Create a new DB test_default. - * 3) Connect to test_default database. - * 4) Connect and create table under test_default_test. - * 5) Connect and display all tables. - * 6) Connect to default database and shouldn't find table test_default_test. - * 7) Connect and drop test_default_test. - * 8) drop test_default database. + hs2Conn = getConnection(jdbcUri + ";", System.getProperty("user.name"),"bar"); + verifyCurrentDB("default", hs2Conn); + hs2Conn.close(); + + hs2Conn = getConnection(jdbcUri + ";/foo=bar;foo1=bar1", System.getProperty("user.name"),"bar"); + verifyCurrentDB("default", hs2Conn); + hs2Conn.close(); + } + + @Test + public void testConnectionSchemaAPIs() throws Exception { + String db1 = "DB1"; + /** + * get/set Schema are new in JDK7 and not available in java.sql.Connection in JDK6. + * Hence the test uses HiveConnection object to call these methods so that test will run with older JDKs */ + HiveConnection hiveConn = (HiveConnection)hs2Conn; + + assertEquals("default", hiveConn.getSchema()); + Statement stmt = hs2Conn.createStatement(); + stmt.execute("DROP DATABASE IF EXISTS " + db1 + " CASCADE"); + stmt.execute("CREATE DATABASE " + db1); + assertEquals("default", hiveConn.getSchema()); - @Test - public void testURIDatabaseName() throws Exception{ - - String jdbcUri = miniHS2.getJdbcURL().substring(0, miniHS2.getJdbcURL().indexOf("default")); - - hs2Conn= getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar"); - String dbName="test_connection_non_default_db"; - String tableInNonDefaultSchema="table_in_non_default_schema"; - Statement stmt = hs2Conn.createStatement(); - stmt.execute("create database if not exists "+dbName); - stmt.close(); - hs2Conn.close(); - - hs2Conn = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar"); - stmt = hs2Conn .createStatement(); - boolean expected = stmt.execute(" create table "+tableInNonDefaultSchema +" (x int)"); - stmt.close(); - hs2Conn .close(); - - hs2Conn = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar"); - stmt = hs2Conn .createStatement(); - ResultSet res = stmt.executeQuery("show tables"); - boolean testTableExists = false; - while (res.next()) { - assertNotNull("table name is null in result set", res.getString(1)); - if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) { - testTableExists = true; - } - } - assertTrue("table name "+tableInNonDefaultSchema - + " found in SHOW TABLES result set", testTableExists); - stmt.close(); - hs2Conn .close(); - - hs2Conn = getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar"); - stmt = hs2Conn .createStatement(); - res = stmt.executeQuery("show tables"); - testTableExists = false; - while (res.next()) { - assertNotNull("table name is null in result set", res.getString(1)); - if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) { - testTableExists = true; - } - } - - assertFalse("table name "+tableInNonDefaultSchema - + " NOT found in SHOW TABLES result set", testTableExists); - stmt.close(); - hs2Conn .close(); - - hs2Conn = getConnection(jdbcUri+dbName,System.getProperty("user.name"),"bar"); - stmt = hs2Conn .createStatement(); - stmt.execute("set hive.support.concurrency = false"); - res = stmt.executeQuery("show tables"); - - stmt.execute(" drop table if exists table_in_non_default_schema"); - expected = stmt.execute("DROP DATABASE "+ dbName); - stmt.close(); - - hs2Conn = getConnection(jdbcUri+"default",System.getProperty("user.name"),"bar"); - stmt = hs2Conn .createStatement(); - res = stmt.executeQuery("show tables"); - testTableExists = false; - while (res.next()) { - assertNotNull("table name is null in result set", res.getString(1)); - if (tableInNonDefaultSchema.equalsIgnoreCase(res.getString(1))) { - testTableExists = true; - } - } - - // test URI with no dbName - hs2Conn = getConnection(jdbcUri, System.getProperty("user.name"),"bar"); - verifyCurrentDB("default", hs2Conn); - hs2Conn.close(); - - hs2Conn = getConnection(jdbcUri + ";", System.getProperty("user.name"),"bar"); - verifyCurrentDB("default", hs2Conn); - hs2Conn.close(); - - hs2Conn = getConnection(jdbcUri + ";/foo=bar;foo1=bar1", System.getProperty("user.name"),"bar"); - verifyCurrentDB("default", hs2Conn); - hs2Conn.close(); - } - - @Test - public void testConnectionSchemaAPIs() throws Exception { - String db1 = "DB1"; - /** - * get/set Schema are new in JDK7 and not available in java.sql.Connection in JDK6. - * Hence the test uses HiveConnection object to call these methods so that test will run with older JDKs - */ - HiveConnection hiveConn = (HiveConnection)hs2Conn; - - assertEquals("default", hiveConn.getSchema()); - Statement stmt = hs2Conn.createStatement(); - stmt.execute("DROP DATABASE IF EXISTS " + db1 + " CASCADE"); - stmt.execute("CREATE DATABASE " + db1); - assertEquals("default", hiveConn.getSchema()); - - stmt.execute("USE " + db1); - assertEquals(db1, hiveConn.getSchema()); - - stmt.execute("USE default"); - assertEquals("default", hiveConn.getSchema()); - - hiveConn.setSchema(db1); - assertEquals(db1, hiveConn.getSchema()); - hiveConn.setSchema("default"); - assertEquals("default", hiveConn.getSchema()); - - assertTrue(hiveConn.getCatalog().isEmpty()); - hiveConn.setCatalog("foo"); - assertTrue(hiveConn.getCatalog().isEmpty()); - } - - /** - * verify that the current db is the one expected. first create table as .tab and then - * describe that table to check if is the current database - * @param expectedDbName - * @param hs2Conn - * @throws Exception - */ - private void verifyCurrentDB(String expectedDbName, Connection hs2Conn) throws Exception { - String verifyTab = "miniHS2DbVerificationTable"; - Statement stmt = hs2Conn.createStatement(); - stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab); - stmt.execute("CREATE TABLE " + expectedDbName + "." + verifyTab + "(id INT)"); - stmt.execute("DESCRIBE " + verifyTab); - stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab); - stmt.close(); - } - - /** - * This method tests whether while creating a new connection, the config - * variables specified in the JDBC URI are properly set for the connection. - * This is a test for HiveConnection#configureConnection. - * - * @throws Exception - */ - @Test - public void testNewConnectionConfiguration() throws Exception { - - // Set some conf parameters - String hiveConf = "hive.cli.print.header=true;hive.server2.async.exec.shutdown.timeout=20;" - + "hive.server2.async.exec.threads=30;hive.server2.thrift.http.max.worker.threads=15"; - // Set some conf vars - String hiveVar = "stab=salesTable;icol=customerID"; - String jdbcUri = miniHS2.getJdbcURL() + "?" + hiveConf + "#" + hiveVar; - - // Open a new connection with these conf & vars - Connection con1 = DriverManager.getConnection(jdbcUri); - - // Execute "set" command and retrieve values for the conf & vars specified - // above - // Assert values retrieved - Statement stmt = con1.createStatement(); - - // Verify that the property has been properly set while creating the - // connection above - verifyConfProperty(stmt, "hive.cli.print.header", "true"); - verifyConfProperty(stmt, "hive.server2.async.exec.shutdown.timeout", "20"); - verifyConfProperty(stmt, "hive.server2.async.exec.threads", "30"); - verifyConfProperty(stmt, "hive.server2.thrift.http.max.worker.threads", - "15"); - verifyConfProperty(stmt, "stab", "salesTable"); - verifyConfProperty(stmt, "icol", "customerID"); - con1.close(); - } - - private void verifyConfProperty(Statement stmt, String property, - String expectedValue) throws Exception { - ResultSet res = stmt.executeQuery("set " + property); - while (res.next()) { - String resultValues[] = res.getString(1).split("="); - assertEquals(resultValues[1], expectedValue); - } - } + stmt.execute("USE " + db1); + assertEquals(db1, hiveConn.getSchema()); + + stmt.execute("USE default"); + assertEquals("default", hiveConn.getSchema()); + + hiveConn.setSchema(db1); + assertEquals(db1, hiveConn.getSchema()); + hiveConn.setSchema("default"); + assertEquals("default", hiveConn.getSchema()); + + assertTrue(hiveConn.getCatalog().isEmpty()); + hiveConn.setCatalog("foo"); + assertTrue(hiveConn.getCatalog().isEmpty()); + } + + /** + * verify that the current db is the one expected. first create table as .tab and then + * describe that table to check if is the current database + * @param expectedDbName + * @param hs2Conn + * @throws Exception + */ + private void verifyCurrentDB(String expectedDbName, Connection hs2Conn) throws Exception { + String verifyTab = "miniHS2DbVerificationTable"; + Statement stmt = hs2Conn.createStatement(); + stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab); + stmt.execute("CREATE TABLE " + expectedDbName + "." + verifyTab + "(id INT)"); + stmt.execute("DESCRIBE " + verifyTab); + stmt.execute("DROP TABLE IF EXISTS " + expectedDbName + "." + verifyTab); + stmt.close(); + } + + /** + * This method tests whether while creating a new connection, the config + * variables specified in the JDBC URI are properly set for the connection. + * This is a test for HiveConnection#configureConnection. + * + * @throws Exception + */ + @Test + public void testNewConnectionConfiguration() throws Exception { + + // Set some conf parameters + String hiveConf = "hive.cli.print.header=true;hive.server2.async.exec.shutdown.timeout=20;" + + "hive.server2.async.exec.threads=30;hive.server2.thrift.http.max.worker.threads=15"; + // Set some conf vars + String hiveVar = "stab=salesTable;icol=customerID"; + String jdbcUri = miniHS2.getJdbcURL() + "?" + hiveConf + "#" + hiveVar; + + // Open a new connection with these conf & vars + Connection con1 = DriverManager.getConnection(jdbcUri); + + // Execute "set" command and retrieve values for the conf & vars specified + // above + // Assert values retrieved + Statement stmt = con1.createStatement(); + + // Verify that the property has been properly set while creating the + // connection above + verifyConfProperty(stmt, "hive.cli.print.header", "true"); + verifyConfProperty(stmt, "hive.server2.async.exec.shutdown.timeout", "20"); + verifyConfProperty(stmt, "hive.server2.async.exec.threads", "30"); + verifyConfProperty(stmt, "hive.server2.thrift.http.max.worker.threads", + "15"); + verifyConfProperty(stmt, "stab", "salesTable"); + verifyConfProperty(stmt, "icol", "customerID"); + con1.close(); + } + + private void verifyConfProperty(Statement stmt, String property, + String expectedValue) throws Exception { + ResultSet res = stmt.executeQuery("set " + property); + while (res.next()) { + String resultValues[] = res.getString(1).split("="); + assertEquals(resultValues[1], expectedValue); + } + } + + /** + * Tests the creation of the 3 scratch dirs: hdfs, local, downloaded resources (which is also local). + * Create a new miniHS2 instance with dfs.permissions.enabled=false, so that we can test with doAs=true + * 1. Test with doAs=false: open a new JDBC session and verify the presence of directories/permissions + * 2. Test with doAs=true: open a new JDBC session and verify the presence of directories/permissions + * @throws Exception + */ + @Test + public void testScratchDirs() throws Exception { + // Stop HiveServer2 + if (miniHS2.isStarted()) { + miniHS2.stop(); + } + HiveConf conf = new HiveConf(); + String userName; + Path scratchDirPath; + conf.setBoolean("dfs.permissions.enabled", false); + // 1. Test with doAs=false + conf.setBoolean("hive.server2.enable.doAs", false); + // Set a custom prefix for hdfs scratch dir path + conf.set("hive.exec.scratchdir.prefix", "/tmp/hs2-"); + // Set a scratch dir permission + String fsPermissionStr = "700"; + conf.set("hive.scratch.dir.permission", fsPermissionStr); + // Start an instance of HiveServer2 which uses miniMR + miniHS2 = new MiniHS2(conf); + Map confOverlay = new HashMap(); + miniHS2.start(confOverlay); + userName = System.getProperty("user.name"); + hs2Conn = getConnection(miniHS2.getJdbcURL(), userName, "password"); + // FS + FileSystem fs = miniHS2.getLocalFS(); + + // Verify scratch dir paths and permission + // HDFS scratch dir + scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR_PREFIX) + + userName); + verifyScratchDir(conf, fs, scratchDirPath, userName, false); + + // Local scratch dir + scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR)); + verifyScratchDir(conf, fs, scratchDirPath, userName, true); + + // Downloaded resources dir + scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR)); + verifyScratchDir(conf, fs, scratchDirPath, userName, true); + + // 2. Test with doAs=true + // Restart HiveServer2 with doAs=true + if (miniHS2.isStarted()) { + miniHS2.stop(); + } + conf.setBoolean("hive.server2.enable.doAs", true); + // Start HS2 + miniHS2 = new MiniHS2(conf); + miniHS2.start(confOverlay); + userName = "neo"; + hs2Conn = getConnection(miniHS2.getJdbcURL(), userName, "the-one"); + + // Verify scratch dir paths and permission + // HDFS scratch dir + scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR_PREFIX) + + userName); + verifyScratchDir(conf, fs, scratchDirPath, userName, false); + + // Local scratch dir + scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR)); + verifyScratchDir(conf, fs, scratchDirPath, userName, true); + + // Downloaded resources dir + scratchDirPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR)); + verifyScratchDir(conf, fs, scratchDirPath, userName, true); + } + + private void verifyScratchDir(HiveConf conf, FileSystem fs, Path scratchDirPath, + String userName, boolean isLocal) throws Exception { + String dirType = isLocal ? "Local" : "DFS"; + FsPermission expectedFSPermission = new FsPermission(HiveConf.getVar(conf, + HiveConf.ConfVars.SCRATCHDIRPERMISSION)); + assertTrue("The expected " + dirType + " scratch dir does not exist for the user: " + + userName, fs.exists(scratchDirPath)); + if (fs.exists(scratchDirPath) && !isLocal) { + assertEquals("The expected DFS scratch dir permissions: " + expectedFSPermission.toString() + + " do not match", expectedFSPermission, fs.getFileStatus(scratchDirPath).getPermission()); + } } +} \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java index 668e580..7fcbe3c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java @@ -18,6 +18,18 @@ package org.apache.hadoop.hive.ql; +import java.io.DataInput; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; + import org.antlr.runtime.TokenRewriteStream; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -30,7 +42,6 @@ import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.TaskRunner; -import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.lockmgr.HiveLock; import org.apache.hadoop.hive.ql.lockmgr.HiveLockManager; @@ -41,18 +52,6 @@ import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.util.StringUtils; -import java.io.DataInput; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.URI; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.ConcurrentHashMap; - /** * Context for Semantic Analyzers. Usage: not reusable - construct a new one for * each query should call clear() at end of use to remove temporary folders @@ -191,7 +190,7 @@ public String getCmd () { * @param scratchDir path of tmp directory */ private Path getScratchDir(String scheme, String authority, - boolean mkdir, String scratchDir) { + boolean mkdir, String scratchDir) { String fileSystem = scheme + ":" + authority; Path dir = fsScratchDirs.get(fileSystem + "-" + TaskRunner.getTaskRunnerID()); @@ -203,11 +202,11 @@ private Path getScratchDir(String scheme, String authority, try { FileSystem fs = dirPath.getFileSystem(conf); dirPath = new Path(fs.makeQualified(dirPath).toString()); - FsPermission fsPermission = new FsPermission(Short.parseShort(scratchDirPermission.trim(), 8)); + FsPermission fsPermission = new FsPermission(scratchDirPermission); - if (!Utilities.createDirsWithPermission(conf, dirPath, fsPermission)) { + if (!fs.mkdirs(dirPath, fsPermission)) { throw new RuntimeException("Cannot make directory: " - + dirPath.toString()); + + dirPath.toString()); } if (isHDFSCleanup) { fs.deleteOnExit(dirPath); @@ -233,7 +232,7 @@ public Path getLocalScratchDir(boolean mkdir) { FileSystem fs = FileSystem.getLocal(conf); URI uri = fs.getUri(); return getScratchDir(uri.getScheme(), uri.getAuthority(), - mkdir, localScratchDir); + mkdir, localScratchDir); } catch (IOException e) { throw new RuntimeException (e); } @@ -257,7 +256,7 @@ public Path getMRScratchDir() { URI uri = dir.toUri(); Path newScratchDir = getScratchDir(uri.getScheme(), uri.getAuthority(), - !explain, uri.getPath()); + !explain, uri.getPath()); LOG.info("New scratch dir is " + newScratchDir); return newScratchDir; } catch (IOException e) { @@ -270,7 +269,7 @@ public Path getMRScratchDir() { private Path getExternalScratchDir(URI extURI) { return getScratchDir(extURI.getScheme(), extURI.getAuthority(), - !explain, nonLocalScratchPath.toUri().getPath()); + !explain, nonLocalScratchPath.toUri().getPath()); } /** @@ -283,7 +282,7 @@ private void removeScratchDir() { p.getFileSystem(conf).delete(p, true); } catch (Exception e) { LOG.warn("Error Removing Scratch: " - + StringUtils.stringifyException(e)); + + StringUtils.stringifyException(e)); } } fsScratchDirs.clear(); @@ -305,7 +304,7 @@ private String nextPathId() { */ public boolean isMRTmpFileURI(String uriStr) { return (uriStr.indexOf(executionId) != -1) && - (uriStr.indexOf(MR_PREFIX) != -1); + (uriStr.indexOf(MR_PREFIX) != -1); } /** @@ -315,7 +314,7 @@ public boolean isMRTmpFileURI(String uriStr) { */ public Path getMRTmpPath() { return new Path(getMRScratchDir(), MR_PREFIX + - nextPathId()); + nextPathId()); } /** @@ -343,7 +342,7 @@ public Path getExternalTmpPath(Path path) { return getExtTmpPathRelTo(path.getParent()); } return new Path(getExternalScratchDir(extURI), EXT_PREFIX + - nextPathId()); + nextPathId()); } /** @@ -353,8 +352,8 @@ public Path getExternalTmpPath(Path path) { */ public Path getExtTmpPathRelTo(Path path) { URI uri = path.toUri(); - return new Path (getScratchDir(uri.getScheme(), uri.getAuthority(), !explain, - uri.getPath() + Path.SEPARATOR + "_" + this.executionId), EXT_PREFIX + nextPathId()); + return new Path (getScratchDir(uri.getScheme(), uri.getAuthority(), !explain, + uri.getPath() + Path.SEPARATOR + "_" + this.executionId), EXT_PREFIX + nextPathId()); } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index d5324b2..5537914 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -92,7 +92,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; -import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.common.HiveInterruptCallback; import org.apache.hadoop.hive.common.HiveInterruptUtils; import org.apache.hadoop.hive.common.HiveStatsUtils; @@ -3352,7 +3351,6 @@ public static void createTmpDirs(Configuration conf, ReduceWork rWork) private static void createTmpDirs(Configuration conf, List> ops) throws IOException { - FsPermission fsPermission = new FsPermission((short)00777); while (!ops.isEmpty()) { Operator op = ops.remove(0); @@ -3362,7 +3360,8 @@ private static void createTmpDirs(Configuration conf, if (tempDir != null) { Path tempPath = Utilities.toTempPath(tempDir); - createDirsWithPermission(conf, tempPath, fsPermission); + FileSystem fs = tempPath.getFileSystem(conf); + fs.mkdirs(tempPath); } } @@ -3498,76 +3497,6 @@ public static int getFooterCount(TableDesc table, JobConf job) throws IOExceptio } /** - * @param conf the configuration used to derive the filesystem to create the path - * @param mkdir the path to be created - * @param fsPermission ignored if it is hive server session and doAs is enabled - * @return true if successfully created the directory else false - * @throws IOException if hdfs experiences any error conditions - */ - public static boolean createDirsWithPermission(Configuration conf, Path mkdir, - FsPermission fsPermission) throws IOException { - - boolean recursive = false; - if (SessionState.get() != null) { - recursive = SessionState.get().isHiveServerQuery() && - conf.getBoolean(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, - HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.defaultBoolVal); - // we reset the permission in case of hive server and doAs enabled because - // currently scratch directory uses /tmp/hive-hive as the scratch directory. - // However, with doAs enabled, the first user to create this directory would - // own the directory and subsequent users cannot access the scratch directory. - // The right fix is to have scratch dir per user. - fsPermission = new FsPermission((short)00777); - } - - // if we made it so far without exception we are good! - return createDirsWithPermission(conf, mkdir, fsPermission, recursive); - } - - private static void resetConfAndCloseFS (Configuration conf, boolean unsetUmask, - String origUmask, FileSystem fs) throws IOException { - if (unsetUmask) { - if (origUmask != null) { - conf.set(FsPermission.UMASK_LABEL, origUmask); - } else { - conf.unset(FsPermission.UMASK_LABEL); - } - } - - fs.close(); - } - - public static boolean createDirsWithPermission(Configuration conf, Path mkdirPath, - FsPermission fsPermission, boolean recursive) throws IOException { - String origUmask = null; - LOG.debug("Create dirs " + mkdirPath + " with permission " + fsPermission + " recursive " + - recursive); - - if (recursive) { - origUmask = conf.get(FsPermission.UMASK_LABEL); - // this umask is required because by default the hdfs mask is 022 resulting in - // all parents getting the fsPermission & !(022) permission instead of fsPermission - conf.set(FsPermission.UMASK_LABEL, "000"); - } - - FileSystem fs = ShimLoader.getHadoopShims().getNonCachedFileSystem(mkdirPath.toUri(), conf); - boolean retval = false; - try { - retval = fs.mkdirs(mkdirPath, fsPermission); - resetConfAndCloseFS(conf, recursive, origUmask, fs); - } catch (IOException ioe) { - try { - resetConfAndCloseFS(conf, recursive, origUmask, fs); - } - catch (IOException e) { - // do nothing - double failure - } - } - return retval; - } - - - /** * Convert path to qualified path. * * @param conf diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java index 2959fcc..9c42a15 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java @@ -41,12 +41,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.ErrorMsg; -import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -303,8 +299,8 @@ private Path createTezDir(String sessionId) Path tezDir = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR), TEZ_DIR); tezDir = new Path(tezDir, sessionId); FileSystem fs = tezDir.getFileSystem(conf); - FsPermission fsPermission = new FsPermission((short)00777); - Utilities.createDirsWithPermission(conf, tezDir, fsPermission, true); + FsPermission fsPermission = new FsPermission(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION)); + fs.mkdirs(tezDir, fsPermission); // Make sure the path is normalized (we expect validation to pass since we just created it). tezDir = DagUtils.validateTargetDir(tezDir, conf).getPath(); // don't keep the directory around on non-clean exit diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 9798cf3..86e11c8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -67,6 +67,7 @@ import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveMetastoreClientFactoryImpl; import org.apache.hadoop.hive.ql.util.DosToUnix; import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; import com.google.common.base.Preconditions; @@ -184,7 +185,7 @@ private PerfLogger perfLogger; - private final String userName; + private String userName; /** * scratch path to use for all non-local (ie. hdfs) file system tmp folders @@ -340,36 +341,42 @@ public static SessionState start(SessionState startSs) { setCurrentSessionState(startSs); - if(startSs.hiveHist == null){ + if (startSs.hiveHist == null){ if (startSs.getConf().getBoolVar(HiveConf.ConfVars.HIVE_SESSION_HISTORY_ENABLED)) { startSs.hiveHist = new HiveHistoryImpl(startSs); - }else { - //Hive history is disabled, create a no-op proxy + } else { + // Hive history is disabled, create a no-op proxy startSs.hiveHist = HiveHistoryProxyHandler.getNoOpHiveHistoryProxy(); } } - if (startSs.getTmpOutputFile() == null) { - // set temp file containing results to be sent to HiveClient - try { - startSs.setTmpOutputFile(createTempFile(startSs.getConf())); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - // Get the following out of the way when you start the session these take a // while and should be done when we start up. try { - //Hive object instance should be created with a copy of the conf object. If the conf is + // Hive object instance should be created with a copy of the conf object. If the conf is // shared with SessionState, other parts of the code might update the config, but // Hive.get(HiveConf) would not recognize the case when it needs refreshing Hive.get(new HiveConf(startSs.conf)).getMSC(); - ShimLoader.getHadoopShims().getUGIForConf(startSs.conf); + UserGroupInformation sessionUGI = ShimLoader.getHadoopShims().getUGIForConf(startSs.conf); FileSystem.get(startSs.conf); - startSs.createSessionPaths(startSs.conf); + + // Create scratch dirs for this session + startSs.createSessionScratchDirs(sessionUGI.getShortUserName()); + + // Create session paths for this session + startSs.createSessionPaths(); + + // Set temp file containing results to be sent to HiveClient + if (startSs.getTmpOutputFile() == null) { + try { + startSs.setTmpOutputFile(createTempFile(startSs.getConf())); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } catch (Exception e) { - // catch-all due to some exec time dependencies on session state + // Catch-all due to some exec time dependencies on session state // that would cause ClassNoFoundException otherwise throw new RuntimeException(e); } @@ -444,42 +451,87 @@ private void dropSessionPaths(Configuration conf) throws IOException { } } - private void createSessionPaths(Configuration conf) throws IOException { - + private void createSessionPaths() throws IOException { + Configuration conf = getConf(); String scratchDirPermission = HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION); String sessionId = getSessionId(); // local & non-local tmp location is configurable. however it is the same across // all external file systems hdfsSessionPath = - new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR), - sessionId); - createPath(conf, hdfsSessionPath, scratchDirPermission); + new Path(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR), + sessionId); + createPath(conf, hdfsSessionPath, scratchDirPermission, false); conf.set(HDFS_SESSION_PATH_KEY, hdfsSessionPath.toUri().toString()); localSessionPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR), - sessionId); - createPath(conf, localSessionPath, scratchDirPermission); + sessionId); + createPath(conf, localSessionPath, scratchDirPermission, true); conf.set(LOCAL_SESSION_PATH_KEY, localSessionPath.toUri().toString()); hdfsTmpTableSpace = new Path(hdfsSessionPath, TMP_PREFIX); - createPath(conf, hdfsTmpTableSpace, scratchDirPermission); + createPath(conf, hdfsTmpTableSpace, scratchDirPermission, false); conf.set(TMP_TABLE_SPACE_KEY, hdfsTmpTableSpace.toUri().toString()); } - private void createPath(Configuration conf, Path p, String perm) throws IOException { - FileSystem fs = p.getFileSystem(conf); + private void createPath(Configuration conf, Path p, String perm, boolean isLocal) throws IOException { + FileSystem fs; + if (isLocal) { + fs = FileSystem.getLocal(conf); + } else { + fs = p.getFileSystem(conf); + } p = new Path(fs.makeQualified(p).toString()); - FsPermission fsPermission = new FsPermission(Short.parseShort(perm.trim(), 8)); + FsPermission fsPermission = new FsPermission(perm); - if (!Utilities.createDirsWithPermission(conf, p, fsPermission)) { + if (!fs.mkdirs(p, fsPermission)) { throw new IOException("Cannot create directory: " - + p.toString()); + + p.toString()); } // best effort to clean up if we don't shut down properly fs.deleteOnExit(p); } + /** + * Create dirs for this session: + * 1. HDFS scratch dir + * 2. Local scratch dir + * 3. Local downloaded resource dir + * @param conf + * @throws IOException + */ + private void createSessionScratchDirs(String userName) throws IOException { + HiveConf conf = getConf(); + String scratchDirPermission = HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION); + // HDFS scratch dir + String hdfsScratchDir = HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR_PREFIX) + userName; + conf.setVar(HiveConf.ConfVars.SCRATCHDIR, hdfsScratchDir); + setupScratchDir(hdfsScratchDir, scratchDirPermission, false, conf); + // Local scratch dir + String localScratchDir = HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR); + setupScratchDir(localScratchDir, scratchDirPermission, true, conf); + // Download resources dir + String downloadResourceDir = HiveConf.getVar(conf, HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR); + setupScratchDir(downloadResourceDir, scratchDirPermission, true, conf); + } + + // Create the give Path if doesn't exists + private void setupScratchDir(String dirPath, String scratchDirPermission, boolean isLocal, + HiveConf conf) throws IOException { + Path scratchDirPath = new Path(dirPath); + FsPermission fsPermission = new FsPermission(scratchDirPermission); + FileSystem fs; + if (isLocal) { + fs = FileSystem.getLocal(conf); + } else { + fs = scratchDirPath.getFileSystem(conf); + } + if (!fs.exists(scratchDirPath)) { + fs.mkdirs(scratchDirPath, fsPermission); + String dirType = isLocal ? "local" : "HDFS"; + LOG.info("Created " + dirType + " directory: " + scratchDirPath.toString()); + } + } /** * Setup authentication and authorization plugins for this session. diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java index 7fd6c17..390a4a6 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestUtilities.java @@ -20,15 +20,12 @@ import static org.apache.hadoop.hive.ql.exec.Utilities.getFileExtension; -import java.io.IOException; import java.sql.Timestamp; import java.util.ArrayList; import java.util.List; import junit.framework.TestCase; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.metadata.HiveException; @@ -39,7 +36,6 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFFromUtcTimestamp; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; import org.apache.hadoop.mapred.JobConf; -import org.junit.Test; public class TestUtilities extends TestCase { @@ -77,9 +73,9 @@ public void testSerializeTimestamp() { List children = new ArrayList(1); children.add(constant); ExprNodeGenericFuncDesc desc = new ExprNodeGenericFuncDesc(TypeInfoFactory.timestampTypeInfo, - new GenericUDFFromUtcTimestamp(), children); + new GenericUDFFromUtcTimestamp(), children); assertEquals(desc.getExprString(), Utilities.deserializeExpression( - Utilities.serializeExpression(desc)).getExprString()); + Utilities.serializeExpression(desc)).getExprString()); } public void testgetDbTableName() throws HiveException{ @@ -109,23 +105,4 @@ public void testgetDbTableName() throws HiveException{ assertEquals("Invalid table name " + tablename, ex.getMessage()); } } - - @Test - public void testFSUmaskReset() throws Exception { - // ensure that FS Umask is not reset (HIVE-7001) - checkFSUMaskReset(true); - checkFSUMaskReset(false); - } - - private void checkFSUMaskReset(boolean recursiveArg) throws IllegalArgumentException, IOException { - final String FS_MASK_VAL = "055"; - HiveConf conf = new HiveConf(); - String dir = System.getProperty("test.tmp.dir") + "/testUtilitiesUMaskReset"; - conf.set(FsPermission.UMASK_LABEL, FS_MASK_VAL); - Utilities.createDirsWithPermission(conf, new Path(dir), new FsPermission((short) 00777), - recursiveArg); - assertEquals(conf.get(FsPermission.UMASK_LABEL), FS_MASK_VAL); - } - - } diff --git a/service/src/java/org/apache/hive/service/cli/CLIService.java b/service/src/java/org/apache/hive/service/cli/CLIService.java index d2cdfc1..422cd9b 100644 --- a/service/src/java/org/apache/hive/service/cli/CLIService.java +++ b/service/src/java/org/apache/hive/service/cli/CLIService.java @@ -30,12 +30,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.conf.SystemVariables; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -46,7 +42,6 @@ import org.apache.hive.service.ServiceException; import org.apache.hive.service.auth.HiveAuthFactory; import org.apache.hive.service.cli.operation.Operation; -import org.apache.hive.service.cli.session.HiveSession; import org.apache.hive.service.cli.session.SessionManager; import org.apache.hive.service.cli.thrift.TProtocolVersion; @@ -123,16 +118,6 @@ public UserGroupInformation getHttpUGI() { @Override public synchronized void start() { super.start(); - - try { - // make sure that the base scratch directories exists and writable - setupStagingDir(hiveConf.getVar(HiveConf.ConfVars.SCRATCHDIR), false); - setupStagingDir(hiveConf.getVar(HiveConf.ConfVars.LOCALSCRATCHDIR), true); - setupStagingDir(hiveConf.getVar(HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR), true); - } catch (IOException eIO) { - throw new ServiceException("Error setting stage directories", eIO); - } - try { // Initialize and test a connection to the metastore metastoreClient = new HiveMetaStoreClient(hiveConf); @@ -467,25 +452,6 @@ public synchronized String getDelegationTokenFromMetaStore(String owner) } } - // create the give Path if doesn't exists and make it writable - private void setupStagingDir(String dirPath, boolean isLocal) throws IOException { - Path scratchDir = getStaticPath(new Path(dirPath)); - if (scratchDir == null) { - return; - } - FileSystem fs; - if (isLocal) { - fs = FileSystem.getLocal(hiveConf); - } else { - fs = scratchDir.getFileSystem(hiveConf); - } - if (!fs.exists(scratchDir)) { - fs.mkdirs(scratchDir); - } - FsPermission fsPermission = new FsPermission((short)0777); - fs.setPermission(scratchDir, fsPermission); - } - @Override public String getDelegationToken(SessionHandle sessionHandle, HiveAuthFactory authFactory, String owner, String renewer) throws HiveSQLException { @@ -509,16 +475,4 @@ public void renewDelegationToken(SessionHandle sessionHandle, HiveAuthFactory au sessionManager.getSession(sessionHandle).renewDelegationToken(authFactory, tokenStr); LOG.info(sessionHandle + ": renewDelegationToken()"); } - - // DOWNLOADED_RESOURCES_DIR for example, which is by default ${system:java.io.tmpdir}/${hive.session.id}_resources, - // {system:java.io.tmpdir} would be already evaluated but ${hive.session.id} would be not in here. - // for that case, this returns evaluatd parts only, in this case, "/tmp" - // what for ${hive.session.id}_resources/${system:java.io.tmpdir}? just don't do that. - private Path getStaticPath(Path path) { - Path current = path; - for (; current != null && SystemVariables.containsVar(current.getName()); - current = current.getParent()) { - } - return current; - } } diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java index eee1cc6..848258e 100644 --- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java +++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java @@ -18,14 +18,14 @@ package org.apache.hive.service.cli.session; +import java.util.Map; + import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.service.cli.SessionHandle; import org.apache.hive.service.cli.operation.OperationManager; import org.apache.hive.service.cli.thrift.TProtocolVersion; -import java.util.Map; - /** * Methods that don't need to be executed under a doAs * context are here. Rest of them in HiveSession interface diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java index bc0a02c..26dfb39 100644 --- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java +++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java @@ -73,7 +73,7 @@ private String username; private final String password; - private final HiveConf hiveConf; + private HiveConf hiveConf; private final SessionState sessionState; private String ipAddress; @@ -206,6 +206,12 @@ public void setOperationManager(OperationManager operationManager) { } @Override + /** + * Opens a new HiveServer2 session for the client connection. + * Note that if doAs is true, this call goes through a proxy object, + * which wraps the method logic in a UserGroupInformation#doAs. + * That is why it is important to call SessionState#start here rather than the constructor. + */ public void open() { SessionState.start(sessionState); } diff --git a/service/src/java/org/apache/hive/service/cli/session/SessionManager.java b/service/src/java/org/apache/hive/service/cli/session/SessionManager.java index d573592..34f169d 100644 --- a/service/src/java/org/apache/hive/service/cli/session/SessionManager.java +++ b/service/src/java/org/apache/hive/service/cli/session/SessionManager.java @@ -120,11 +120,13 @@ public SessionHandle openSession(TProtocolVersion protocol, String username, Str Map sessionConf, boolean withImpersonation, String delegationToken) throws HiveSQLException { HiveSession session; + // If doAs is set to true for HiveServer2, we will create a proxy object for the session impl. + // Within the proxy object, we wrap the method call in a UserGroupInformation#doAs if (withImpersonation) { - HiveSessionImplwithUGI hiveSessionUgi = new HiveSessionImplwithUGI(protocol, username, password, - hiveConf, ipAddress, delegationToken); - session = HiveSessionProxy.getProxy(hiveSessionUgi, hiveSessionUgi.getSessionUgi()); - hiveSessionUgi.setProxySession(session); + HiveSessionImplwithUGI sessionWithUGI = new HiveSessionImplwithUGI(protocol, username, password, + hiveConf, ipAddress, delegationToken); + session = HiveSessionProxy.getProxy(sessionWithUGI, sessionWithUGI.getSessionUgi()); + sessionWithUGI.setProxySession(session); } else { session = new HiveSessionImpl(protocol, username, password, hiveConf, ipAddress); } diff --git a/service/src/test/org/apache/hive/service/cli/TestScratchDir.java b/service/src/test/org/apache/hive/service/cli/TestScratchDir.java deleted file mode 100644 index bf96b96..0000000 --- a/service/src/test/org/apache/hive/service/cli/TestScratchDir.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hive.service.cli; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hive.service.cli.thrift.EmbeddedThriftBinaryCLIService; -import org.apache.hive.service.cli.thrift.ThriftCLIService; -import org.apache.hive.service.cli.thrift.ThriftCLIServiceClient; -import org.junit.Test; - -public class TestScratchDir { - @Test - public void testScratchDirs() throws Exception { - stageDirTest("hive.exec.scratchdir", "TestScratchDirs_foobar", false); - } - - @Test - public void testLocalScratchDirs() throws Exception { - stageDirTest("hive.exec.local.scratchdir", "TestLocalScratchDirs_foobar", true); - } - - @Test - public void testResourceDirs() throws Exception { - stageDirTest("hive.downloaded.resources.dir", "TestResourceDirs_foobar", true); - } - - private void stageDirTest(String stageDirConfigStr, String stageDirName, boolean isLocal) throws IOException { - String scratchDirStr = System.getProperty("test.tmp.dir") + File.separator + - stageDirName; - System.setProperty(stageDirConfigStr, scratchDirStr); - ThriftCLIService service = new EmbeddedThriftBinaryCLIService(); - ThriftCLIServiceClient client = new ThriftCLIServiceClient(service); - final Path scratchDir = new Path(scratchDirStr); - Configuration conf = new Configuration(); - FileSystem fs = scratchDir.getFileSystem(conf); - if (isLocal) { - fs = FileSystem.getLocal(conf); - } - assertTrue(fs.exists(scratchDir)); - - FileStatus[] fStatus = fs.globStatus(scratchDir); - boolean foo = fStatus[0].equals(new FsPermission((short)0777)); - assertEquals(new FsPermission((short)0777), fStatus[0].getPermission()); - service.stop(); - fs.delete(scratchDir, true); - System.clearProperty(stageDirConfigStr); - } - -}