diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 7c88f4f..88523e3 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1457,6 +1457,8 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal HIVETEZLOGLEVEL("hive.tez.log.level", "INFO", "The log level to use for tasks executing as part of the DAG.\n" + "Used only if hive.tez.java.opts is used to configure Java options."), + HIVETEZHS2USERACCESS("hive.tez.hs2.user.access", true, + "Whether to grant access to the hs2/hive user for queries"), HIVEQUERYNAME ("hive.query.name", null, "This named is used by Tez to set the dag name. This name in turn will appear on \n" + "the Tez UI representing the work that was done."), diff --git common/src/java/org/apache/hive/common/util/ACLConfigurationParser.java common/src/java/org/apache/hive/common/util/ACLConfigurationParser.java index e69de29..a9a9e01 100644 --- common/src/java/org/apache/hive/common/util/ACLConfigurationParser.java +++ common/src/java/org/apache/hive/common/util/ACLConfigurationParser.java @@ -0,0 +1,167 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.common.util; + +import java.util.Collections; +import java.util.Set; +import java.util.regex.Pattern; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.conf.Configuration; + +import com.google.common.collect.Sets; + +/** + * Parser for extracting ACL information from Configs + */ +@Private +public class ACLConfigurationParser { + + private static final Logger LOG = + LoggerFactory.getLogger(ACLConfigurationParser.class); + + private static final String WILDCARD_ACL_VALUE = "*"; + private static final Pattern splitPattern = Pattern.compile("\\s+"); + + private final Set allowedUsers; + private final Set allowedGroups; + + public ACLConfigurationParser(Configuration conf, String confPropertyName) { + allowedUsers = Sets.newLinkedHashSet(); + allowedGroups = Sets.newLinkedHashSet(); + parse(conf, confPropertyName); + } + + + private boolean isWildCard(String aclStr) { + return aclStr.trim().equals(WILDCARD_ACL_VALUE); + } + + private void parse(Configuration conf, String configProperty) { + String aclsStr = conf.get(configProperty); + if (aclsStr == null || aclsStr.isEmpty()) { + return; + } + if (isWildCard(aclsStr)) { + allowedUsers.add(WILDCARD_ACL_VALUE); + return; + } + + final String[] splits = splitPattern.split(aclsStr); + int counter = -1; + String userListStr = null; + String groupListStr = null; + for (String s : splits) { + if (s.isEmpty()) { + if (userListStr != null) { + continue; + } + } + ++counter; + if (counter == 0) { + userListStr = s; + } else if (counter == 1) { + groupListStr = s; + } else { + LOG.warn("Invalid configuration specified for " + configProperty + + ", ignoring configured ACLs, value=" + aclsStr); + return; + } + } + + if (userListStr == null) { + return; + } + if (userListStr.length() >= 1) { + allowedUsers.addAll( + org.apache.hadoop.util.StringUtils.getTrimmedStringCollection(userListStr)); + } + if (groupListStr != null && groupListStr.length() >= 1) { + allowedGroups.addAll( + org.apache.hadoop.util.StringUtils.getTrimmedStringCollection(userListStr)); + } + } + + public Set getAllowedUsers() { + return Collections.unmodifiableSet(allowedUsers); + } + + public Set getAllowedGroups() { + return Collections.unmodifiableSet(allowedGroups); + } + + public void addAllowedUser(String user) { + if (StringUtils.isBlank(user)) { + return; + } + if (allowedUsers.contains(WILDCARD_ACL_VALUE)) { + return; + } + if (user.equals(WILDCARD_ACL_VALUE)) { + allowedUsers.clear(); + allowedGroups.clear(); + } + allowedUsers.add(user); + } + + public void addAllowedGroup(String group) { + allowedGroups.add(group); + } + + public String toAclString() { + return toString(); + } + + @Override + public String toString() { + if (getAllowedUsers().contains(WILDCARD_ACL_VALUE)) { + return WILDCARD_ACL_VALUE; + } else { + if (allowedUsers.size() == 0 && allowedGroups.size() == 0) { + return " "; + } + String userString = constructCsv(allowedUsers); + String groupString = ""; + if (allowedGroups.size() > 0) { + groupString = " " + constructCsv(allowedGroups); + } + return userString + groupString; + } + } + + private String constructCsv(Set inSet) { + StringBuilder sb = new StringBuilder(); + if (inSet != null) { + boolean isFirst = true; + for (String s : inSet) { + if (!isFirst) { + sb.append(","); + } else { + isFirst = false; + } + sb.append(s); + } + } + return sb.toString(); + } + +} diff --git common/src/test/org/apache/hive/common/util/TestACLConfigurationParser.java common/src/test/org/apache/hive/common/util/TestACLConfigurationParser.java index e69de29..c50768a 100644 --- common/src/test/org/apache/hive/common/util/TestACLConfigurationParser.java +++ common/src/test/org/apache/hive/common/util/TestACLConfigurationParser.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.common.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.conf.Configuration; +import org.junit.Test; + +public class TestACLConfigurationParser { + + + @Test (timeout = 10_000L) + public void test() { + + ACLConfigurationParser aclConf; + Configuration conf = new Configuration(); + conf.set("ACL_ALL_ACCESS", "*"); + aclConf = new ACLConfigurationParser(conf, "ACL_ALL_ACCESS"); + assertEquals(1, aclConf.getAllowedUsers().size()); + assertTrue(aclConf.getAllowedUsers().contains("*")); + assertEquals(0, aclConf.getAllowedGroups().size()); + assertEquals("*", aclConf.toAclString()); + + conf.set("ACL_INVALID1", "u1, u2, u3"); + aclConf = new ACLConfigurationParser(conf, "ACL_INVALID1"); + assertEquals(0, aclConf.getAllowedUsers().size()); + assertEquals(0, aclConf.getAllowedGroups().size()); + assertEquals(" ", aclConf.toAclString()); + +// conf.set("ACL_INVALID2", "u1, u2 "); +// aclConf = new ACLConfigurationParser(conf, "ACL_INVALID2"); +// assertEquals(0, aclConf.getAllowedUsers().size()); +// assertEquals(0, aclConf.getAllowedGroups().size()); +// assertEquals(" ", aclConf.toAclString()); + + conf.set("ACL_NONE", " "); + aclConf = new ACLConfigurationParser(conf, "ACL_NONE"); + assertEquals(0, aclConf.getAllowedUsers().size()); + assertEquals(0, aclConf.getAllowedGroups().size()); + assertEquals(" ", aclConf.toAclString()); + + conf.set("ACL_VALID1", "user1,user2"); + aclConf = new ACLConfigurationParser(conf, "ACL_VALID1"); + assertEquals(2, aclConf.getAllowedUsers().size()); + assertTrue(aclConf.getAllowedUsers().contains("user1")); + assertTrue(aclConf.getAllowedUsers().contains("user2")); + assertEquals(0, aclConf.getAllowedGroups().size()); + assertEquals("user1,user2", aclConf.toAclString()); + + conf.set("ACL_VALID2", "user1,user2 group1,group2"); + aclConf = new ACLConfigurationParser(conf, "ACL_VALID2"); + assertEquals(2, aclConf.getAllowedUsers().size()); + assertTrue(aclConf.getAllowedUsers().contains("user1")); + assertTrue(aclConf.getAllowedUsers().contains("user2")); + assertEquals(2, aclConf.getAllowedGroups().size()); + assertTrue(aclConf.getAllowedGroups().contains("group1")); + assertTrue(aclConf.getAllowedGroups().contains("group2")); + assertEquals("user1,user2 group1,group2", aclConf.toAclString()); + + + conf.set("ACL_VALID3", "user1 group1"); + aclConf = new ACLConfigurationParser(conf, "ACL_VALID3"); + assertEquals(1, aclConf.getAllowedUsers().size()); + assertTrue(aclConf.getAllowedUsers().contains("user1")); + assertEquals(1, aclConf.getAllowedGroups().size()); + assertTrue(aclConf.getAllowedGroups().contains("group1")); + assertEquals("user1 group1", aclConf.toAclString()); + + aclConf.addAllowedUser("user2"); + assertEquals(2, aclConf.getAllowedUsers().size()); + assertTrue(aclConf.getAllowedUsers().contains("user1")); + assertTrue(aclConf.getAllowedUsers().contains("user2")); + assertEquals("user1,user2 group1", aclConf.toAclString()); + + aclConf.addAllowedGroup("group2"); + assertEquals(2, aclConf.getAllowedGroups().size()); + assertTrue(aclConf.getAllowedGroups().contains("group1")); + assertTrue(aclConf.getAllowedGroups().contains("group2")); + assertEquals("user1,user2 group1,group2", aclConf.toAclString()); + + aclConf.addAllowedUser("*"); + assertEquals(1, aclConf.getAllowedUsers().size()); + assertTrue(aclConf.getAllowedUsers().contains("*")); + assertTrue(aclConf.getAllowedGroups().isEmpty()); + } + +} diff --git ql/src/java/org/apache/hadoop/hive/ql/Driver.java ql/src/java/org/apache/hadoop/hive/ql/Driver.java index 592b1f1..cdf24d4 100644 --- ql/src/java/org/apache/hadoop/hive/ql/Driver.java +++ ql/src/java/org/apache/hadoop/hive/ql/Driver.java @@ -1747,7 +1747,8 @@ public int execute(boolean deferClose) throws CommandNeedRetryException { resStream = null; SessionState ss = SessionState.get(); - hookContext = new HookContext(plan, queryState, ctx.getPathToCS(), ss.getUserName(), + + hookContext = new HookContext(plan, queryState, ctx.getPathToCS(), ss.getUserFromAuthenticator(), ss.getUserIpAddress(), InetAddress.getLocalHost().getHostAddress(), operationId, ss.getSessionId(), Thread.currentThread().getName(), ss.isHiveServerQuery(), perfLogger); hookContext.setHookType(HookContext.HookType.PRE_EXEC_HOOK); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index e81cbce..3484493 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -143,6 +143,7 @@ import org.apache.hadoop.mapred.SequenceFileOutputFormat; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.Shell; +import org.apache.hive.common.util.ACLConfigurationParser; import org.apache.hive.common.util.ReflectionUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -3794,4 +3795,26 @@ public static String humanReadableByteCount(long bytes) { String suffix = "KMGTPE".charAt(exp-1) + ""; return String.format("%.2f%sB", bytes / Math.pow(unit, exp), suffix); } + + + public static String getAclStringWithHiveModification(Configuration tezConf, + String propertyName, + boolean addHs2User, + String user, + String hs2User) throws + IOException { + + // Start with initial ACLs + ACLConfigurationParser aclConf = + new ACLConfigurationParser(tezConf, propertyName); + + // Always give access to the user + aclConf.addAllowedUser(user); + + // Give access to the process user if the config is set. + if (addHs2User && hs2User != null) { + aclConf.addAllowedUser(hs2User); + } + return aclConf.toAclString(); + } } diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java index 62f65c2..ed1ba9c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java @@ -308,6 +308,8 @@ protected void openInternal(final HiveConf conf, Collection additionalFi tezConfig.setInt(TezConfiguration.TEZ_AM_SESSION_MIN_HELD_CONTAINERS, n); } + setupSessionAcls(tezConfig, conf); + final TezClient session = TezClient.newBuilder("HIVE-" + sessionId, tezConfig) .setIsSession(true).setLocalResources(commonLocalResources) .setCredentials(llapCredentials).setServicePluginDescriptor(servicePluginsDescriptor) @@ -433,6 +435,31 @@ public void endOpen() throws InterruptedException, CancellationException { } } + private void setupSessionAcls(Configuration tezConf, HiveConf hiveConf) throws + IOException { + + String user = SessionState.getUserFromAuthenticator(); + UserGroupInformation loginUserUgi = UserGroupInformation.getLoginUser(); + String loginUser = + loginUserUgi == null ? null : loginUserUgi.getShortUserName(); + boolean addHs2User = + HiveConf.getBoolVar(hiveConf, ConfVars.HIVETEZHS2USERACCESS); + + String viewStr = Utilities.getAclStringWithHiveModification(tezConf, + TezConfiguration.TEZ_AM_VIEW_ACLS, addHs2User, user, loginUser); + String modifyStr = Utilities.getAclStringWithHiveModification(tezConf, + TezConfiguration.TEZ_AM_MODIFY_ACLS, addHs2User, user, loginUser); + + if (LOG.isDebugEnabled()) { + LOG.debug( + "Setting Tez Session access for sessionId={} with viewAclString={}, modifyStr={}", + SessionState.get().getSessionId(), viewStr, modifyStr); + } + + tezConf.set(TezConfiguration.TEZ_AM_VIEW_ACLS, viewStr); + tezConf.set(TezConfiguration.TEZ_AM_MODIFY_ACLS, modifyStr); + } + public void refreshLocalResourcesFromConf(HiveConf conf) throws IOException, LoginException, IllegalArgumentException, URISyntaxException, TezException { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index 58f0b33..740e41b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -33,6 +33,7 @@ import javax.annotation.Nullable; import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.metrics.common.Metrics; @@ -57,6 +58,7 @@ import org.apache.hadoop.hive.ql.plan.api.StageType; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -71,6 +73,7 @@ import org.apache.tez.dag.api.Edge; import org.apache.tez.dag.api.GroupInputEdge; import org.apache.tez.dag.api.SessionNotRunning; +import org.apache.tez.dag.api.TezConfiguration; import org.apache.tez.dag.api.TezException; import org.apache.tez.dag.api.Vertex; import org.apache.tez.dag.api.VertexGroup; @@ -348,7 +351,7 @@ DAG build(JobConf conf, TezWork work, Path scratchDir, dag.setDAGInfo(dagInfo); dag.setCredentials(conf.getCredentials()); - setAccessControlsForCurrentUser(dag); + setAccessControlsForCurrentUser(dag, queryPlan.getQueryId(), conf); for (BaseWork w: ws) { @@ -431,14 +434,31 @@ DAG build(JobConf conf, TezWork work, Path scratchDir, return dag; } - public static void setAccessControlsForCurrentUser(DAG dag) { - // get current user - String currentUser = SessionState.getUserFromAuthenticator(); - if(LOG.isDebugEnabled()) { - LOG.debug("Setting Tez DAG access for " + currentUser); + private static void setAccessControlsForCurrentUser(DAG dag, String queryId, + Configuration conf) throws + IOException { + String user = SessionState.getUserFromAuthenticator(); + UserGroupInformation loginUserUgi = UserGroupInformation.getLoginUser(); + String loginUser = + loginUserUgi == null ? null : loginUserUgi.getShortUserName(); + boolean addHs2User = + HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVETEZHS2USERACCESS); + + // Temporarily re-using the TEZ AM View ACLs property for individual dag access control. + // Hive may want to setup it's own parameters if it wants to control per dag access. + // Setting the tez-property per dag should work for now. + + String viewStr = Utilities.getAclStringWithHiveModification(conf, + TezConfiguration.TEZ_AM_VIEW_ACLS, addHs2User, user, loginUser); + String modifyStr = Utilities.getAclStringWithHiveModification(conf, + TezConfiguration.TEZ_AM_MODIFY_ACLS, addHs2User, user, loginUser); + + if (LOG.isDebugEnabled()) { + LOG.debug("Setting Tez DAG access for queryId={} with viewAclString={}, modifyStr={}", + queryId, viewStr, modifyStr); } // set permissions for current user on DAG - DAGAccessControls ac = new DAGAccessControls(currentUser, currentUser); + DAGAccessControls ac = new DAGAccessControls(viewStr, modifyStr); dag.setAccessControls(ac); } diff --git ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java index 72a1acc..13ccd93 100644 --- ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java +++ ql/src/java/org/apache/hadoop/hive/ql/hooks/ATSHook.java @@ -162,7 +162,7 @@ public void run() { String queryId = plan.getQueryId(); String opId = hookContext.getOperationId(); long queryStartTime = plan.getQueryStartTime(); - String user = hookContext.getUgi().getUserName(); + String user = hookContext.getUgi().getShortUserName(); String requestuser = hookContext.getUserName(); if (hookContext.getUserName() == null ){ requestuser = hookContext.getUgi().getUserName() ;