diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index e2bd38b4e1..2e4eb9c447 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -794,6 +794,9 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal METASTORE_EVENT_DB_LISTENER_TTL("hive.metastore.event.db.listener.timetolive", "86400s", new TimeValidator(TimeUnit.SECONDS), "time after which events will be removed from the database listener queue"), + METASTORE_EVENT_DB_NOTIFICATION_API_AUTH("hive.metastore.event.db.notification.api.auth", true, + "Should metastore do authorization against database notification related APIs such as get_next_notification.\n" + + "If set to true, then only the superusers in proxy settings have the permission"), METASTORE_AUTHORIZATION_STORAGE_AUTH_CHECKS("hive.metastore.authorization.storage.checks", false, "Should the metastore do authorization checks against the underlying storage (usually hdfs) \n" + "for operations like drop-partition (disallow the drop-partition if the user in\n" + diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java index d2474cc1ac..d375dd3fc3 100644 --- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java +++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java @@ -49,6 +49,7 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe; +import org.apache.hadoop.hive.shims.Utils; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hive.hcatalog.api.repl.Command; import org.apache.hive.hcatalog.api.repl.ReplicationTask; @@ -123,6 +124,7 @@ public static void startMetaStoreServer() throws Exception { hcatConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, ""); hcatConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); + hcatConf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " "); } diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/repl/commands/TestCommands.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/repl/commands/TestCommands.java index b9593218ae..072f1f91d7 100644 --- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/repl/commands/TestCommands.java +++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/repl/commands/TestCommands.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.shims.Utils; import org.apache.hive.hcatalog.HcatTestUtils; import org.apache.hive.hcatalog.api.HCatAddPartitionDesc; import org.apache.hive.hcatalog.api.HCatClient; @@ -80,6 +81,7 @@ public static void setUpBeforeClass() throws Exception { hconf .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); + hconf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); TEST_PATH = System.getProperty("test.warehouse.dir","/tmp") + Path.SEPARATOR + TestCommands.class.getCanonicalName() + "-" + System.currentTimeMillis(); Path testPath = new Path(TEST_PATH); diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java index b9a32180ee..a5fc13a173 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/api/TestHCatClientNotification.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.NotificationEvent; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.hive.shims.Utils; import org.apache.hive.hcatalog.common.HCatConstants; import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; import org.apache.hive.hcatalog.listener.DbNotificationListener; @@ -68,6 +69,7 @@ public static void setupClient() throws Exception { HiveConf conf = new HiveConf(); conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS, DbNotificationListener.class.getName()); + conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); hCatClient = HCatClient.create(conf); md = MessageFactory.getInstance().getDeserializer(); } diff --git a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java index c36b632fd1..5da2286070 100644 --- a/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java +++ b/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/TestDbNotificationListener.java @@ -96,8 +96,10 @@ import org.apache.hadoop.hive.metastore.messaging.MessageFactory; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.shims.Utils; import org.apache.hive.hcatalog.api.repl.ReplicationV1CompatRule; import org.apache.hive.hcatalog.data.Pair; +import org.apache.thrift.TException; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; @@ -117,9 +119,12 @@ private static final int EVENTS_TTL = 30; private static final int CLEANUP_SLEEP_TIME = 10; private static Map emptyParameters = new HashMap(); + private static HiveConf conf; private static IMetaStoreClient msClient; private static Driver driver; private static MessageDeserializer md = null; + private static String proxySettingName = null; + private int startTime; private long firstEventId; @@ -232,7 +237,7 @@ public void onInsert(InsertEvent insertEvent) throws MetaException { @SuppressWarnings("rawtypes") @BeforeClass public static void connectToMetastore() throws Exception { - HiveConf conf = new HiveConf(); + conf = new HiveConf(); conf.setVar(HiveConf.ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS, DbNotificationListener.class.getName()); conf.setVar(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS, MockMetaStoreEventListener.class.getName()); @@ -253,6 +258,10 @@ public static void connectToMetastore() throws Exception { } conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); + + proxySettingName = "hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts"; + conf.set(proxySettingName, "*"); + SessionState.start(new CliSessionState(conf)); msClient = new HiveMetaStoreClient(conf); driver = new Driver(conf); @@ -1575,4 +1584,38 @@ public void cleanupNotifs() throws Exception { LOG.info("second trigger done"); assertEquals(0, rsp2.getEventsSize()); } + + @Test(expected = TException.class) + public void testAuthForNotificationAPIs() throws Exception { + // Setup + String dbName = "testAuthForNotificationAPIs"; + driver.run("create database " + dbName); + // Test the getNextNotification API + try { + NotificationEventResponse rsp = msClient.getNextNotification(firstEventId, 0, null); + assertEquals(1, rsp.getEventsSize()); + // Remove the proxy privilege + conf.unset(proxySettingName); + // Disable auth so it should still work + conf.setBoolVar(HiveConf.ConfVars.METASTORE_EVENT_DB_NOTIFICATION_API_AUTH, false); + msClient.close(); + msClient = new HiveMetaStoreClient(conf); + rsp = msClient.getNextNotification(firstEventId, 0, null); + assertEquals(1, rsp.getEventsSize()); + // Turn auth back on, which should fail + conf.setBoolVar(HiveConf.ConfVars.METASTORE_EVENT_DB_NOTIFICATION_API_AUTH, true); + msClient.close(); + msClient = new HiveMetaStoreClient(conf); + rsp = msClient.getNextNotification(firstEventId, 0, null); + } catch (Exception ex) { + throw ex; + } finally { + conf.setBoolVar(HiveConf.ConfVars.METASTORE_EVENT_DB_NOTIFICATION_API_AUTH, true); + conf.set(proxySettingName, "*"); + msClient.close(); + msClient = new HiveMetaStoreClient(conf); + driver.close(); + driver = new Driver(conf); + } + } } diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestCopyUtils.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestCopyUtils.java index 98b2a3cfa9..f14b4300da 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestCopyUtils.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestCopyUtils.java @@ -81,12 +81,12 @@ public static void classLevelSetup() throws Exception { Configuration conf = new Configuration(); conf.set("dfs.client.use.datanode.hostname", "true"); - MiniDFSCluster miniDFSCluster = - new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build(); - UserGroupInformation ugi = Utils.getUGI(); - String currentUser = ugi.getShortUserName(); + final String currentUser = ugi.getShortUserName(); + conf.set("hadoop.proxyuser." + currentUser + ".hosts", "*"); + MiniDFSCluster miniDFSCluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build(); HashMap overridesForHiveConf = new HashMap() {{ put(ConfVars.HIVE_IN_TEST.varname, "false"); put(ConfVars.HIVE_EXEC_COPYFILE_MAXSIZE.varname, "1"); diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java index 70a57f833c..e86ee5e961 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestExportImport.java @@ -21,6 +21,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.shims.Utils; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -48,6 +49,7 @@ public static void classLevelSetup() throws Exception { Configuration conf = new Configuration(); conf.set("dfs.client.use.datanode.hostname", "true"); + conf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); MiniDFSCluster miniDFSCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).build(); HashMap overridesForHiveConf = new HashMap() {{ diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index a8c3a0b49e..beaea8648f 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hive.ql.exec.repl.ReplDumpWork; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.hive.shims.Utils; import org.apache.hive.hcatalog.api.repl.ReplicationV1CompatRule; import org.apache.thrift.TException; import org.junit.After; @@ -144,6 +145,7 @@ public static void setUpBeforeClass() throws Exception { hconf.set(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL.varname, "org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore"); hconf.setBoolVar(HiveConf.ConfVars.HIVEOPTIMIZEMETADATAQUERIES, true); + hconf.set("hadoop.proxyuser." + Utils.getUGI().getShortUserName() + ".hosts", "*"); System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " "); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloTestSetup.java b/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloTestSetup.java index 47cf7ac79a..366a7de69b 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloTestSetup.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/accumulo/AccumuloTestSetup.java @@ -48,6 +48,7 @@ protected MiniAccumuloCluster miniCluster; public AccumuloTestSetup() { + System.setProperty("hive.metastore.event.db.notification.api.auth", "false"); } protected void setupWithHiveConf(HiveConf conf) throws Exception { diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java index 67e03a4c24..494492e2b5 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/AbstractCoreBlobstoreCliDriver.java @@ -51,6 +51,7 @@ public AbstractCoreBlobstoreCliDriver(AbstractCliConfig testCliConfig) { @Override @BeforeClass public void beforeClass() { + System.setProperty("hive.metastore.event.db.notification.api.auth", "false"); MiniClusterType miniMR = cliConfig.getClusterType(); String hiveConfDir = cliConfig.getHiveConfDir(); String initScript = cliConfig.getInitScript(); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java index a1762ec46d..a0d08122db 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreCliDriver.java @@ -48,6 +48,7 @@ public CoreCliDriver(AbstractCliConfig testCliConfig) { @Override @BeforeClass public void beforeClass() { + System.setProperty("hive.metastore.event.db.notification.api.auth", "false"); String message = "Starting " + CoreCliDriver.class.getName() + " run at " + System.currentTimeMillis(); LOG.info(message); System.err.println(message); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java index 794798839c..63aa0143fa 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CoreNegativeCliDriver.java @@ -37,6 +37,7 @@ public CoreNegativeCliDriver(AbstractCliConfig testCliConfig) { @Override public void beforeClass(){ + System.setProperty("hive.metastore.event.db.notification.api.auth", "false"); MiniClusterType miniMR = cliConfig.getClusterType(); String hiveConfDir = cliConfig.getHiveConfDir(); String initScript = cliConfig.getInitScript(); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java index d80bd44cb5..0c7c8c2171 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/cli/control/CorePerfCliDriver.java @@ -53,6 +53,7 @@ public CorePerfCliDriver(AbstractCliConfig testCliConfig) { public void beforeClass() { System.setProperty("datanucleus.schema.autoCreateAll", "true"); System.setProperty("hive.metastore.schema.verification", "false"); + System.setProperty("hive.metastore.event.db.notification.api.auth", "false"); MiniClusterType miniMR = cliConfig.getClusterType(); String hiveConfDir = cliConfig.getHiveConfDir(); String initScript = cliConfig.getInitScript(); diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index d5de4f2c7b..e7258a137d 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -7055,12 +7055,28 @@ private Table getTable(String dbName, String tableName) @Override public NotificationEventResponse get_next_notification(NotificationEventRequest rqst) throws TException { + try { + authorizeProxyPrivilege(); + } catch (Exception ex) { + LOG.error("Not authorized to make the get_next_notification call. You can try to disable " + + HiveConf.ConfVars.METASTORE_EVENT_DB_NOTIFICATION_API_AUTH.varname, ex); + throw new TException(ex); + } + RawStore ms = getMS(); return ms.getNextNotification(rqst); } @Override public CurrentNotificationEventId get_current_notificationEventId() throws TException { + try { + authorizeProxyPrivilege(); + } catch (Exception ex) { + LOG.error("Not authorized to make the get_current_notificationEventId call. You can try to disable " + + HiveConf.ConfVars.METASTORE_EVENT_DB_NOTIFICATION_API_AUTH.varname, ex); + throw new TException(ex); + } + RawStore ms = getMS(); return ms.getCurrentNotificationEventId(); } @@ -7068,10 +7084,34 @@ public CurrentNotificationEventId get_current_notificationEventId() throws TExce @Override public NotificationEventsCountResponse get_notification_events_count(NotificationEventsCountRequest rqst) throws TException { + try { + authorizeProxyPrivilege(); + } catch (Exception ex) { + LOG.error("Not authorized to make the get_notification_events_count call. You can try to disable " + + HiveConf.ConfVars.METASTORE_EVENT_DB_NOTIFICATION_API_AUTH.varname, ex); + throw new TException(ex); + } + RawStore ms = getMS(); return ms.getNotificationEventsCount(rqst); } + private void authorizeProxyPrivilege() throws Exception + { + if (!hiveConf.getBoolVar(HiveConf.ConfVars.METASTORE_EVENT_DB_NOTIFICATION_API_AUTH)) { + return; + } + try { + String user = Utils.getUGI().getShortUserName(); + if (!MetaStoreUtils.hasPermissionForDbNotificationCalls(user, hiveConf, getIPAddress())) { + throw new MetaException("User " + user + " is not allowed to perform this API call"); + } + } catch (Exception ex) { + LOG.error("Cannot obtain username", ex); + throw ex; + } + } + @Override public FireEventResponse fire_listener_event(FireEventRequest rqst) throws TException { switch (rqst.getData().getSetField()) { diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index b51446d5c6..e4e5ae37b1 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -31,6 +31,7 @@ import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -106,6 +107,9 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.security.SaslRpcServer; +import org.apache.hadoop.security.authorize.DefaultImpersonationProvider; +import org.apache.hadoop.security.authorize.ProxyUsers; +import org.apache.hadoop.util.MachineList; import org.apache.hive.common.util.HiveStringUtils; import org.apache.hive.common.util.ReflectionUtil; @@ -1974,4 +1978,22 @@ public ColumnStatisticsObj call() throws Exception { public static double decimalToDouble(Decimal decimal) { return new BigDecimal(new BigInteger(decimal.getUnscaled()), decimal.getScale()).doubleValue(); } + + /** + * Verify if the user is allowed to make DB notification related calls. + * Only the superusers defined in the Hadoop proxy user settings have the permission. + * + * @param user the short user name + * @param config that contains the proxy user settings + * @return if the user has the permission + */ + public static boolean hasPermissionForDbNotificationCalls(String user, Configuration conf, String ipAddress) { + ProxyUsers.refreshSuperUserGroupsConfiguration(conf); + DefaultImpersonationProvider sip = ProxyUsers.getDefaultImpersonationProvider(); + Map> proxyHosts = sip.getProxyHosts(); + Collection hostEntries = proxyHosts.get(sip.getProxySuperuserIpConfKey(user)); + MachineList machineList = new MachineList(hostEntries); + ipAddress = (ipAddress == null) ? StringUtils.EMPTY : ipAddress; + return machineList.includes(ipAddress); + } }