diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index b21d763..fd47a48 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -2411,6 +2411,9 @@ public static boolean isAclEnabled(Configuration conf) { "security.applicationhistory.protocol.acl"; public static final String + YARN_SECURITY_SERVICE_AUTHORIZATION_REFRESHUSERMAPPING_PROTOCOL = + "security.refreshusermapping.protocol.acl"; + public static final String YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL = "security.collector-nodemanager.protocol.acl"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java index 708af3d..59187d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.PrintStream; +import java.net.InetSocketAddress; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -43,6 +44,7 @@ import org.apache.hadoop.ha.HAAdmin; import org.apache.hadoop.ha.HAServiceTarget; import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.security.RefreshUserMappingsProtocol; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.yarn.api.records.DecommissionType; @@ -51,6 +53,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceOption; +import org.apache.hadoop.yarn.client.AHSProxy; import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.client.RMHAServiceTarget; import org.apache.hadoop.yarn.client.util.YarnClientUtils; @@ -120,9 +123,9 @@ + " will immediately decommission if an RM HA failover occurs.")) .put("-refreshNodesResources", new UsageInfo("", "Refresh resources of NodeManagers at the ResourceManager.")) - .put("-refreshSuperUserGroupsConfiguration", new UsageInfo("", + .put("-refreshSuperUserGroupsConfiguration", new UsageInfo("[-timeline]", "Refresh superuser proxy groups mappings")) - .put("-refreshUserToGroupsMappings", new UsageInfo("", + .put("-refreshUserToGroupsMappings", new UsageInfo("[-timeline]", "Refresh user-to-groups mappings")) .put("-refreshAdminAcls", new UsageInfo("", "Refresh acls for administration of ResourceManager")) @@ -273,8 +276,8 @@ private static void printHelp(String cmd, boolean isHAEnabled) { + " [-refreshQueues]" + " [-refreshNodes [-g|graceful [timeout in seconds] -client|server]]" + " [-refreshNodesResources]" - + " [-refreshSuperUserGroupsConfiguration]" - + " [-refreshUserToGroupsMappings]" + + " [-refreshSuperUserGroupsConfiguration] [-timeline]" + + " [-refreshUserToGroupsMappings] [-timeline]" + " [-refreshAdminAcls]" + " [-refreshServiceAcl]" + " [-getGroup [username]]" @@ -350,6 +353,30 @@ private int refreshQueues() throws IOException, YarnException { return 0; } + protected int refreshSuperUserGroupsConfigurationInAHS() throws IOException { + createAHSProxy().refreshSuperUserGroupsConfiguration(); + System.out.println("Refresh superuser groups configuration in Application History Server"); + return 0; + } + + protected int refreshUserGroupsMappingInAHS() throws IOException { + createAHSProxy().refreshUserToGroupsMappings(); + System.out.println("Refresh user groups mapping in Application History Server"); + return 0; + } + + protected RefreshUserMappingsProtocol createAHSProxy() throws IOException { + Configuration conf = getConf(); + InetSocketAddress address = conf.getSocketAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT); + conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY, + conf.get(YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL, "")); + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + return AHSProxy.createAHSProxyWithRefreshUserMappingsProtocol(address, conf, ugi); + } + + private int refreshNodes(boolean graceful) throws IOException, YarnException { // Refresh the nodes ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol(); @@ -744,9 +771,7 @@ public int run(String[] args) throws Exception { // if ("-refreshAdminAcls".equals(cmd) || "-refreshQueues".equals(cmd) || "-refreshNodesResources".equals(cmd) || - "-refreshServiceAcl".equals(cmd) || - "-refreshUserToGroupsMappings".equals(cmd) || - "-refreshSuperUserGroupsConfiguration".equals(cmd)) { + "-refreshServiceAcl".equals(cmd)) { if (args.length != 1) { printUsage(cmd, isHAEnabled); return exitCode; @@ -761,9 +786,17 @@ public int run(String[] args) throws Exception { } else if ("-refreshNodesResources".equals(cmd)) { exitCode = refreshNodesResources(); } else if ("-refreshUserToGroupsMappings".equals(cmd)) { - exitCode = refreshUserToGroupsMappings(); + if(args.length == 2 && "-timeline".equals(args[i])) { + exitCode = refreshUserGroupsMappingInAHS(); + } else { + exitCode = refreshUserToGroupsMappings(); + } } else if ("-refreshSuperUserGroupsConfiguration".equals(cmd)) { - exitCode = refreshSuperUserGroupsConfiguration(); + if(args.length == 2 && "-timeline".equals(args[i])) { + exitCode = refreshSuperUserGroupsConfigurationInAHS(); + } else { + exitCode = refreshSuperUserGroupsConfiguration(); + } } else if ("-refreshAdminAcls".equals(cmd)) { exitCode = refreshAdminAcls(); } else if ("-refreshServiceAcl".equals(cmd)) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AHSProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AHSProxy.java index e2978b7..2d6ed25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AHSProxy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/AHSProxy.java @@ -27,7 +27,13 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.RefreshUserMappingsProtocol; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolClientSideTranslatorPB; +import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolPB; import org.apache.hadoop.yarn.ipc.YarnRPC; @InterfaceAudience.Public @@ -55,4 +61,15 @@ public T run() { } }); } + + public static RefreshUserMappingsProtocol createAHSProxyWithRefreshUserMappingsProtocol( + InetSocketAddress address, Configuration conf, UserGroupInformation ugi) throws IOException { + Class xface = RefreshUserMappingsProtocolPB.class; + RPC.setProtocolEngine(conf, xface, ProtobufRpcEngine.class); + Object proxy = RPC.getProxy(xface, RPC.getProtocolVersion(xface), address, + ugi, conf, NetUtils.getDefaultSocketFactory(conf), 0); + RefreshUserMappingsProtocolPB refreshProxy = (RefreshUserMappingsProtocolPB) proxy; + return new RefreshUserMappingsProtocolClientSideTranslatorPB(refreshProxy); + + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java index ecaf0fa..9c7728f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java @@ -22,11 +22,19 @@ import java.net.InetSocketAddress; import java.util.ArrayList; +import com.google.protobuf.BlockingService; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; -import org.apache.hadoop.ipc.Server; +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.security.Groups; +import org.apache.hadoop.security.RefreshUserMappingsProtocol; import org.apache.hadoop.security.authorize.PolicyProvider; +import org.apache.hadoop.security.authorize.ProxyUsers; +import org.apache.hadoop.security.proto.RefreshUserMappingsProtocolProtos; +import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolPB; +import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolServerSideTranslatorPB; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol; import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest; @@ -59,15 +67,16 @@ import org.apache.hadoop.yarn.server.timeline.security.authorize.TimelinePolicyProvider; import com.google.common.base.Preconditions; +import org.apache.hadoop.yarn.server.util.timeline.TimelineServerUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class ApplicationHistoryClientService extends AbstractService implements - ApplicationHistoryProtocol { + ApplicationHistoryProtocol, RefreshUserMappingsProtocol { private static final Logger LOG = LoggerFactory.getLogger(ApplicationHistoryClientService.class); private ApplicationHistoryManager history; - private Server server; + private RPC.Server server; private InetSocketAddress bindAddress; public ApplicationHistoryClientService(ApplicationHistoryManager history) { @@ -90,8 +99,7 @@ protected void serviceStart() throws Exception { "%s property value should be greater than zero", YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT); - server = - rpc.getServer(ApplicationHistoryProtocol.class, this, + server =(RPC.Server) rpc.getServer(ApplicationHistoryProtocol.class, this, address, conf, null, conf.getInt( YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT)); @@ -102,6 +110,15 @@ protected void serviceStart() throws Exception { refreshServiceAcls(conf, new TimelinePolicyProvider()); } + RPC.setProtocolEngine(conf, RefreshUserMappingsProtocolPB.class, ProtobufRpcEngine.class); + RefreshUserMappingsProtocolServerSideTranslatorPB refreshUserMappingXlator = + new RefreshUserMappingsProtocolServerSideTranslatorPB(this); + BlockingService refreshUserMappingService = RefreshUserMappingsProtocolProtos. + RefreshUserMappingsProtocolService + .newReflectiveBlockingService(refreshUserMappingXlator); + server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, RefreshUserMappingsProtocolPB.class, + refreshUserMappingService); + server.start(); this.bindAddress = conf.updateConnectAddr(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, @@ -232,4 +249,18 @@ public RenewDelegationTokenResponse renewDelegationToken( RenewDelegationTokenRequest request) throws YarnException, IOException { return null; } + + @Override + public void refreshUserToGroupsMappings() throws IOException { + LOG.info("Refresh user groups mapping in Application History Server."); + Groups.getUserToGroupsMappingService().refresh(); + } + + @Override + public void refreshSuperUserGroupsConfiguration() throws IOException { + LOG.info("Refresh superuser groups configuration in Application History Server."); + Configuration conf = new Configuration(); + TimelineServerUtils.processAHSProxyUsersConf(conf); + ProxyUsers.refreshSuperUserGroupsConfiguration(conf); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/authorize/TimelinePolicyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/authorize/TimelinePolicyProvider.java index 9f3a646..be3cb30 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/authorize/TimelinePolicyProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/authorize/TimelinePolicyProvider.java @@ -20,6 +20,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.security.RefreshUserMappingsProtocol; import org.apache.hadoop.security.authorize.PolicyProvider; import org.apache.hadoop.security.authorize.Service; import org.apache.hadoop.yarn.api.ApplicationHistoryProtocolPB; @@ -37,7 +38,10 @@ return new Service[] { new Service( YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONHISTORY_PROTOCOL, - ApplicationHistoryProtocolPB.class) + ApplicationHistoryProtocolPB.class), + new Service( + YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_REFRESHUSERMAPPING_PROTOCOL, + RefreshUserMappingsProtocol.class) }; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/util/timeline/TimelineServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/util/timeline/TimelineServerUtils.java index 15c6d3d..d70d8b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/util/timeline/TimelineServerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/util/timeline/TimelineServerUtils.java @@ -18,16 +18,20 @@ package org.apache.hadoop.yarn.server.util.timeline; +import java.util.HashMap; import java.util.LinkedHashSet; +import java.util.Map; import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.AuthenticationFilterInitializer; +import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilter; import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer; import org.apache.hadoop.yarn.server.timeline.security.TimelineDelgationTokenSecretManagerService; +import static org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer.PREFIX; /** * Set of utility methods to be used across timeline reader and collector. @@ -90,4 +94,25 @@ public static void addTimelineAuthFilter(String initializers, TimelineAuthenticationFilterInitializer.class.getName()); } } + + /** + * Find all configs whose name starts with + * TIMELINE_SERVICE_PROXY_USER_PREFIX, and add a record for each one by + * replacing the prefix with ProxyUsers.CONF_HADOOP_PROXYUSER + */ + public static void processAHSProxyUsersConf(Configuration conf) { + Map rmProxyUsers = new HashMap(); + for (Map.Entry entry : conf) { + String propName = entry.getKey(); + if (propName.startsWith(PREFIX)) { + rmProxyUsers.put(ProxyUsers.CONF_HADOOP_PROXYUSER + "." + + propName.substring(PREFIX + .length()), + entry.getValue()); + } + } + for (Map.Entry entry : rmProxyUsers.entrySet()) { + conf.set(entry.getKey(), entry.getValue()); + } + } }