From 9345414e70a48dccd9b34ebbb7f367e645c02b5b Mon Sep 17 00:00:00 2001 From: vkorukanti Date: Mon, 19 May 2014 19:46:01 -0700 Subject: [PATCH] HIVE-6245: HS2 creates DBs/Tables with wrong ownership when HMS setugi is true --- .../apache/hive/service/TestHS2WithRemoteMS.java | 133 +++++++++++++++++++++ .../hadoop/hive/metastore/MetaStoreUtils.java | 10 +- .../cli/session/HiveSessionImplwithUGI.java | 15 +-- 3 files changed, 150 insertions(+), 8 deletions(-) create mode 100644 itests/hive-unit/src/test/java/org/apache/hive/service/TestHS2WithRemoteMS.java diff --git itests/hive-unit/src/test/java/org/apache/hive/service/TestHS2WithRemoteMS.java itests/hive-unit/src/test/java/org/apache/hive/service/TestHS2WithRemoteMS.java new file mode 100644 index 0000000..5c2d572 --- /dev/null +++ itests/hive-unit/src/test/java/org/apache/hive/service/TestHS2WithRemoteMS.java @@ -0,0 +1,133 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.service; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim; +import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hive.jdbc.miniHS2.MiniHS2; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.HashMap; + +public class TestHS2WithRemoteMS { + + private static MiniHS2 miniHS2 = null; + + @BeforeClass + public static void startServices() throws Exception { + int metaStorePort = MetaStoreUtils.findFreePort(); + + HiveConf hiveConf = new HiveConf(); + hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_MIN_WORKER_THREADS, 1); + hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS, 1); + hiveConf.setBoolVar(ConfVars.METASTORE_EXECUTE_SET_UGI, true); + + // setup metastore uris + hiveConf.setVar(ConfVars.METASTOREURIS, "thrift://localhost:" + metaStorePort); + + hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + miniHS2 = new MiniHS2(hiveConf, true); + + System.out.println("Starting MetaStore Server on port " + metaStorePort); + MetaStoreUtils.startMetaStore(metaStorePort, + ShimLoader.getHadoopThriftAuthBridge(), hiveConf); + + miniHS2.start(new HashMap()); + + MiniDFSShim dfs = miniHS2.getDfs(); + FileSystem fs = dfs.getFileSystem(); + + // change the warehouse directory permissions to 777. + fs.setPermission(miniHS2.getWareHouseDir(), new FsPermission((short)0777)); + } + + @AfterClass + public static void stopServices() throws Exception { + if (miniHS2 != null && miniHS2.isStarted()) { + miniHS2.stop(); + } + } + + @Test + public void testImpersonation() throws Exception { + Class.forName(MiniHS2.getJdbcDriverName()); + + // Create a two tables one with as user "foo" and other as "bar" + Connection hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "foo", null); + + Statement stmt = hs2Conn.createStatement(); + + String tableName = "foo_table"; + stmt.execute("drop table if exists " + tableName); + stmt.execute("create table " + tableName + " (value string)"); + + stmt.close(); + hs2Conn.close(); + + + hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL(), "bar", null); + + stmt = hs2Conn.createStatement(); + + tableName = "bar_table"; + stmt.execute("drop table if exists " + tableName); + stmt.execute("create table " + tableName + " (value string)"); + + stmt.close(); + hs2Conn.close(); + + MiniDFSShim dfs = miniHS2.getDfs(); + FileSystem fs = dfs.getFileSystem(); + + FileStatus[] files = fs.listStatus(miniHS2.getWareHouseDir()); + boolean fooTableValidated = false; + boolean barTableValidated = false; + for(FileStatus file : files) { + if (file.getPath().getName().equals("foo_table")) { + fooTableValidated = file.getOwner().equals("foo"); + assertTrue(String.format("User 'foo' table ownership '%s' is wrong", file.getOwner()), fooTableValidated); + } + else if (file.getPath().getName().equals("bar_table")) { + barTableValidated = file.getOwner().equals("bar"); + assertTrue(String.format("User 'bar' table ownership '%s' is wrong", file.getOwner()), barTableValidated); + } + else + fail(String.format("Unknown table directory '%s' in warehouse", file.getPath().getName())); + + System.out.println(String.format("File: %s, Owner: %s", file.getPath().getName(), file.getOwner())); + } + + assertTrue("User 'foo' table not found in warehouse", fooTableValidated); + assertTrue("User 'bar' table not found in warehouse", barTableValidated); + } +} diff --git metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 5a56ced..d802422 100644 --- metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -48,6 +48,7 @@ import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; @@ -1043,11 +1044,17 @@ public static void makeDir(Path path, HiveConf hiveConf) throws MetaException { public static void startMetaStore(final int port, final HadoopThriftAuthBridge bridge) throws Exception { + startMetaStore(port, bridge, new HiveConf(HMSHandler.class)); + } + + public static void startMetaStore(final int port, + final HadoopThriftAuthBridge bridge, final HiveConf hiveConf) + throws Exception{ Thread thread = new Thread(new Runnable() { @Override public void run() { try { - HiveMetaStore.startMetaStore(port, bridge); + HiveMetaStore.startMetaStore(port, bridge, hiveConf); } catch (Throwable e) { LOG.error("Metastore Thrift Server threw an exception...",e); } @@ -1057,6 +1064,7 @@ public void run() { thread.start(); loopUntilHMSReady(port); } + /** * A simple connect test to make sure that the metastore is up * @throws Exception diff --git service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java index e79b129..b55d08f 100644 --- service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java +++ service/src/java/org/apache/hive/service/cli/session/HiveSessionImplwithUGI.java @@ -49,6 +49,14 @@ public HiveSessionImplwithUGI(TProtocolVersion protocol, String username, String super(protocol, username, password, hiveConf, sessionConf, ipAddress); setSessionUGI(username); setDelegationToken(delegationToken); + + // create a new metastore connection for this particular user session + Hive.set(null); + try { + sessionHive = Hive.get(getHiveConf()); + } catch (HiveException e) { + throw new HiveSQLException("Failed to setup metastore connection", e); + } } // setup appropriate UGI for the session @@ -116,13 +124,6 @@ private void setDelegationToken(String delegationTokenStr) throws HiveSQLExcepti } catch (IOException e) { throw new HiveSQLException("Couldn't setup delegation token in the ugi", e); } - // create a new metastore connection using the delegation token - Hive.set(null); - try { - sessionHive = Hive.get(getHiveConf()); - } catch (HiveException e) { - throw new HiveSQLException("Failed to setup metastore connection", e); - } } } -- 1.8.5.2 (Apple Git-48)