diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java index 4b777ea..0f6468c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java @@ -30,6 +30,14 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.api.protocolrecords.AddLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.AddLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.ClearAllLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.ClearAllLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodeToLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodeToLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest; @@ -42,6 +50,10 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.SetNodeToLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.SetNodeToLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; @@ -110,4 +122,40 @@ public RefreshServiceAclsResponse refreshServiceAcls( public UpdateNodeResourceResponse updateNodeResource( UpdateNodeResourceRequest request) throws YarnException, IOException; + + @Public + @Evolving + @Idempotent + public AddLabelsResponse addLabels(AddLabelsRequest request) + throws YarnException, IOException; + + @Public + @Evolving + @Idempotent + public RemoveLabelsResponse removeLabels( + RemoveLabelsRequest request) throws YarnException, IOException; + + @Public + @Evolving + @Idempotent + public SetNodeToLabelsResponse setNodeToLabels( + SetNodeToLabelsRequest request) throws YarnException, IOException; + + @Public + @Evolving + @Idempotent + public GetNodeToLabelsResponse getNodeToLabels( + GetNodeToLabelsRequest request) throws YarnException, IOException; + + @Public + @Evolving + @Idempotent + public GetLabelsResponse getLabels( + GetLabelsRequest request) throws YarnException, IOException; + + @Public + @Evolving + @Idempotent + public ClearAllLabelsResponse clearAllLabels( + ClearAllLabelsRequest request) throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto index 47a6cf7..2d9fabb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto @@ -39,4 +39,10 @@ service ResourceManagerAdministrationProtocolService { rpc refreshServiceAcls(RefreshServiceAclsRequestProto) returns (RefreshServiceAclsResponseProto); rpc getGroupsForUser(GetGroupsForUserRequestProto) returns (GetGroupsForUserResponseProto); rpc updateNodeResource (UpdateNodeResourceRequestProto) returns (UpdateNodeResourceResponseProto); + rpc addLabels(AddLabelsRequestProto) returns (AddLabelsResponseProto); + rpc removeLabels(RemoveLabelsRequestProto) returns (RemoveLabelsResponseProto); + rpc setNodeToLabels(SetNodeToLabelsRequestProto) returns (SetNodeToLabelsResponseProto); + rpc getNodeToLabels(GetNodeToLabelsRequestProto) returns (GetNodeToLabelsResponseProto); + rpc getLabels(GetLabelsRequestProto) returns (GetLabelsResponseProto); + rpc clearAllLabels(ClearAllLabelsRequestProto) returns (ClearAllLabelsResponseProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java index 50e5825..8fd5f90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java @@ -18,12 +18,23 @@ package org.apache.hadoop.yarn.client.cli; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.net.ConnectException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; -import com.google.common.collect.ImmutableMap; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; @@ -41,13 +52,24 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.label.FileSystemNodeLabelManager; +import org.apache.hadoop.yarn.label.NodeLabelConfiguration; +import org.apache.hadoop.yarn.label.NodeLabelManager; +import org.apache.hadoop.yarn.label.NodeLabelManagerFactory; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; +import org.apache.hadoop.yarn.server.api.protocolrecords.AddLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodeToLabelsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.SetNodeToLabelsRequest; + +import com.google.common.collect.ImmutableMap; @Private @Unstable @@ -55,6 +77,7 @@ private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); + private static final Log LOG = LogFactory.getLog(RMAdminCLI.class); protected final static Map ADMIN_USAGE = ImmutableMap.builder() @@ -78,7 +101,19 @@ .put("-help", new UsageInfo("[cmd]", "Displays help for the given command or all commands if none " + "is specified.")) - .build(); + .put("-addLabels", + new UsageInfo("[labels splitted by ',']", "Add labels")) + .put("-removeLabels", + new UsageInfo("[labels splitted by ',']", "Remove labels")) + .put("-setNodeToLabels", + new UsageInfo("[node1:label1,label2,label3;node2:label2,label3]", + "set node to labels")) + .put("-getNodeToLabels", new UsageInfo("", + "Get node to label mappings")) + .put("-getLabels", new UsageInfo("", "Get labels in the cluster")) + .put("-loadLabelsConfigFile", + new UsageInfo("[path/to to node-label.xml]", "Load labels config file")) + .build(); public RMAdminCLI() { super(); @@ -202,10 +237,26 @@ private static void printUsage(String cmd, boolean isHAEnabled) { } - protected ResourceManagerAdministrationProtocol createAdminProtocol() throws IOException { + protected ResourceManagerAdministrationProtocol createAdminProtocol() + throws IOException { + return createAdminProtocol(false); + } + + protected ResourceManagerAdministrationProtocol + createAdminProtocolDoNotRetry() throws IOException { + return createAdminProtocol(true); + } + + protected ResourceManagerAdministrationProtocol createAdminProtocol( + boolean doNotRetry) throws IOException { // Get the current configuration final YarnConfiguration conf = new YarnConfiguration(getConf()); - return ClientRMProxy.createRMProxy(conf, ResourceManagerAdministrationProtocol.class); + if (doNotRetry) { + conf.setInt("ipc.client.connect.max.retries", 0); + conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 0); + } + return ClientRMProxy.createRMProxy(conf, + ResourceManagerAdministrationProtocol.class); } private int refreshQueues() throws IOException, YarnException { @@ -285,6 +336,180 @@ private int getGroups(String[] usernames) throws IOException { return 0; } + private NodeLabelManager getLocalNodeLabelManagerInstance() { + NodeLabelManager localMgr = + NodeLabelManagerFactory.getNodeLabelManager(getConf()); + if (!(localMgr instanceof FileSystemNodeLabelManager)) { + LOG.warn("Acquired NodeLabelManager is not recoverable, " + + " RMAdmin CLI will exit"); + System.exit(1); + } + + localMgr.init(getConf()); + localMgr.start(); + + return localMgr; + } + + private int addLabels(String args) throws IOException, YarnException { + Set labels = new HashSet(); + for (String p : args.split(",")) { + labels.add(p); + } + + return addLabels(labels); + } + + private int addLabels(Set labels) throws IOException, YarnException { + ResourceManagerAdministrationProtocol adminProtocol = + createAdminProtocolDoNotRetry(); + + try { + AddLabelsRequest request = AddLabelsRequest.newInstance(labels); + adminProtocol.addLabels(request); + } catch (ConnectException e) { + LOG.info("Failed to connect to RM, try to use standalone NodeLabelManager"); + NodeLabelManager mgr = getLocalNodeLabelManagerInstance(); + mgr.persistAddingLabels(labels); + mgr.stop(); + } + + return 0; + } + + private int removeLabels(String args) throws IOException, YarnException { + ResourceManagerAdministrationProtocol adminProtocol = + createAdminProtocolDoNotRetry(); + Set labels = new HashSet(); + for (String p : args.split(",")) { + labels.add(p); + } + + try { + RemoveLabelsRequest request = RemoveLabelsRequest.newInstance(labels); + adminProtocol.removeLabels(request); + } catch (ConnectException e) { + LOG.info("Failed to connect to RM, try to use standalone NodeLabelManager"); + NodeLabelManager mgr = getLocalNodeLabelManagerInstance(); + mgr.persistRemovingLabels(labels); + mgr.stop(); + } + + return 0; + } + + private int getNodeToLabels() throws IOException, YarnException { + ResourceManagerAdministrationProtocol adminProtocol = + createAdminProtocolDoNotRetry(); + + Map> nodeToLabels = null; + try { + nodeToLabels = + adminProtocol.getNodeToLabels(GetNodeToLabelsRequest.newInstance()) + .getNodeToLabels(); + } catch (ConnectException e) { + LOG.info("Failed to connect to RM, try to use standalone NodeLabelManager"); + NodeLabelManager mgr = getLocalNodeLabelManagerInstance(); + nodeToLabels = mgr.getNodesToLabels(); + mgr.stop(); + } + + for (String host : sortSet(nodeToLabels.keySet())) { + System.out.println(String.format("Host=%s, Labels=[%s]", host, + StringUtils.join(sortSet(nodeToLabels.get(host)).iterator(), ","))); + } + return 0; + } + + private int getLabels() throws IOException, YarnException { + ResourceManagerAdministrationProtocol adminProto = + createAdminProtocolDoNotRetry(); + + Set labels = null; + try { + labels = adminProto.getLabels(GetLabelsRequest.newInstance()).getLabels(); + } catch (ConnectException e) { + LOG.info("Failed to connect to RM, try to use standalone NodeLabelManager"); + NodeLabelManager mgr = getLocalNodeLabelManagerInstance(); + labels = mgr.getLabels(); + mgr.stop(); + } + + System.out.println(String.format("Labels=%s", + StringUtils.join(sortSet(labels).iterator(), ","))); + return 0; + } + + private int loadLabelsConfigFile(String configFile) throws IOException, + YarnException { + File file = new File(configFile); + if (!file.exists() || file.isDirectory()) { + LOG.error(String.format("ConfigFile=%s, doesn't exist or it's a dir", + configFile)); + return -1; + } + + NodeLabelConfiguration nodeLabelConfig = + new NodeLabelConfiguration(file.getAbsolutePath()); + + int rc; + if (0 != (rc = addLabels(nodeLabelConfig.getLabels()))) { + return rc; + } + return setNodeToLabels(nodeLabelConfig.getNodeToLabels()); + } + + private List sortSet(Set labels) { + List list = new ArrayList(); + list.addAll(labels); + Collections.sort(list); + return list; + } + + private int setNodeToLabels(String args) throws IOException, YarnException { + Map> map = new HashMap>(); + + for (String nodeToLabels : args.split(";")) { + String[] split = nodeToLabels.split(":"); + if (split.length != 2) { + throw new IOException( + "Format is incorrect, should be node:label1,label2..."); + } + String node = split[0]; + String labels = split[1]; + + if (node.trim().isEmpty()) { + throw new IOException("node name cannot be empty"); + } + + map.put(node, new HashSet()); + for (String label : labels.split(",")) { + if (!label.trim().isEmpty()) { + map.get(node).add(label.trim().toLowerCase()); + } + } + } + + return setNodeToLabels(map); + } + + private int setNodeToLabels(Map> map) throws IOException, + YarnException { + ResourceManagerAdministrationProtocol adminProtocol = + createAdminProtocolDoNotRetry(); + try { + SetNodeToLabelsRequest request = SetNodeToLabelsRequest.newInstance(map); + adminProtocol.setNodeToLabels(request); + } catch (ConnectException e) { + LOG.info("Failed to connect to RM, try to use standalone NodeLabelManager"); + NodeLabelManager mgr = getLocalNodeLabelManagerInstance(); + mgr.persistNodeToLabelsChanges(map); + mgr.stop(); + } + + return 0; + } + @Override public int run(String[] args) throws Exception { YarnConfiguration yarnConf = @@ -351,6 +576,18 @@ public int run(String[] args) throws Exception { } else if ("-getGroups".equals(cmd)) { String[] usernames = Arrays.copyOfRange(args, i, args.length); exitCode = getGroups(usernames); + } else if ("-addLabels".equals(cmd)) { + exitCode = addLabels(args[i]); + } else if ("-removeLabels".equals(cmd)) { + exitCode = removeLabels(args[i]); + } else if ("-setNodeToLabels".equals(cmd)) { + exitCode = setNodeToLabels(args[i]); + } else if ("-getNodeToLabels".equals(cmd)) { + exitCode = getNodeToLabels(); + } else if ("-getLabels".equals(cmd)) { + exitCode = getLabels(); + } else if ("-loadLabelsConfigFile".equals(cmd)) { + exitCode = loadLabelsConfigFile(args[i]); } else { exitCode = -1; System.err.println(cmd.substring(1) + ": Unknown command"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java index ccffaed..1e348fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java @@ -29,17 +29,31 @@ import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.RPCUtil; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddLabelsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ClearAllLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserResponseProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetLabelsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetNodeToLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshAdminAclsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshNodesRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshQueuesRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshServiceAclsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshSuperUserGroupsConfigurationRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshUserToGroupsMappingsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RemoveLabelsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.SetNodeToLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceRequestProto; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB; +import org.apache.hadoop.yarn.server.api.protocolrecords.AddLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.AddLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.ClearAllLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.ClearAllLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodeToLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodeToLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest; @@ -52,8 +66,20 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.SetNodeToLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.SetNodeToLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddLabelsRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddLabelsResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ClearAllLabelsRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ClearAllLabelsResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetLabelsRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetLabelsResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetNodeToLabelsRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetNodeToLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesRequestPBImpl; @@ -66,6 +92,10 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshSuperUserGroupsConfigurationResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshUserToGroupsMappingsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshUserToGroupsMappingsResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RemoveLabelsRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RemoveLabelsResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SetNodeToLabelsRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SetNodeToLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceResponsePBImpl; @@ -205,5 +235,88 @@ public UpdateNodeResourceResponse updateNodeResource( return null; } } + + @Override + public AddLabelsResponse addLabels(AddLabelsRequest request) + throws YarnException, IOException { + AddLabelsRequestProto requestProto = + ((AddLabelsRequestPBImpl) request).getProto(); + try { + return new AddLabelsResponsePBImpl(proxy.addLabels(null, + requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public RemoveLabelsResponse removeLabels( + RemoveLabelsRequest request) throws YarnException, IOException { + RemoveLabelsRequestProto requestProto = + ((RemoveLabelsRequestPBImpl) request).getProto(); + try { + return new RemoveLabelsResponsePBImpl(proxy.removeLabels(null, + requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public SetNodeToLabelsResponse setNodeToLabels( + SetNodeToLabelsRequest request) throws YarnException, IOException { + SetNodeToLabelsRequestProto requestProto = + ((SetNodeToLabelsRequestPBImpl) request).getProto(); + try { + return new SetNodeToLabelsResponsePBImpl(proxy.setNodeToLabels( + null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public GetNodeToLabelsResponse getNodeToLabels(GetNodeToLabelsRequest request) + throws YarnException, IOException { + GetNodeToLabelsRequestProto requestProto = + ((GetNodeToLabelsRequestPBImpl) request).getProto(); + try { + return new GetNodeToLabelsResponsePBImpl(proxy.getNodeToLabels( + null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public GetLabelsResponse getLabels(GetLabelsRequest request) + throws YarnException, IOException { + GetLabelsRequestProto requestProto = + ((GetLabelsRequestPBImpl) request).getProto(); + try { + return new GetLabelsResponsePBImpl(proxy.getLabels( + null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + @Override + public ClearAllLabelsResponse clearAllLabels(ClearAllLabelsRequest request) + throws YarnException, IOException { + ClearAllLabelsRequestProto requestProto = + ((ClearAllLabelsRequestPBImpl) request).getProto(); + try { + return new ClearAllLabelsResponsePBImpl(proxy.clearAllLabels( + null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java index d1f71fe..c6fd8f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java @@ -22,8 +22,16 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddLabelsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddLabelsResponseProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ClearAllLabelsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ClearAllLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserResponseProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetLabelsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetLabelsResponseProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetNodeToLabelsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetNodeToLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshAdminAclsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshAdminAclsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshNodesRequestProto; @@ -36,17 +44,35 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshSuperUserGroupsConfigurationResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshUserToGroupsMappingsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshUserToGroupsMappingsResponseProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RemoveLabelsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RemoveLabelsResponseProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.SetNodeToLabelsRequestProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.SetNodeToLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceResponseProto; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB; +import org.apache.hadoop.yarn.server.api.protocolrecords.AddLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.ClearAllLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodeToLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.SetNodeToLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddLabelsRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddLabelsResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ClearAllLabelsRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ClearAllLabelsResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetLabelsRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetLabelsResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetNodeToLabelsRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetNodeToLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesRequestPBImpl; @@ -59,6 +85,10 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshSuperUserGroupsConfigurationResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshUserToGroupsMappingsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshUserToGroupsMappingsResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RemoveLabelsRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RemoveLabelsResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SetNodeToLabelsRequestPBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.SetNodeToLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceResponsePBImpl; @@ -204,4 +234,94 @@ public UpdateNodeResourceResponseProto updateNodeResource(RpcController controll } } + @Override + public AddLabelsResponseProto addLabels(RpcController controller, + AddLabelsRequestProto proto) throws ServiceException { + AddLabelsRequestPBImpl request = new AddLabelsRequestPBImpl(proto); + try { + AddLabelsResponse response = real.addLabels(request); + return ((AddLabelsResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public RemoveLabelsResponseProto removeLabels( + RpcController controller, RemoveLabelsRequestProto proto) + throws ServiceException { + RemoveLabelsRequestPBImpl request = + new RemoveLabelsRequestPBImpl(proto); + try { + RemoveLabelsResponse response = real.removeLabels(request); + return ((RemoveLabelsResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public SetNodeToLabelsResponseProto setNodeToLabels( + RpcController controller, SetNodeToLabelsRequestProto proto) + throws ServiceException { + SetNodeToLabelsRequestPBImpl request = + new SetNodeToLabelsRequestPBImpl(proto); + try { + SetNodeToLabelsResponse response = real.setNodeToLabels(request); + return ((SetNodeToLabelsResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public GetNodeToLabelsResponseProto getNodeToLabels(RpcController controller, + GetNodeToLabelsRequestProto proto) throws ServiceException { + GetNodeToLabelsRequestPBImpl request = + new GetNodeToLabelsRequestPBImpl(proto); + try { + GetNodeToLabelsResponse response = real.getNodeToLabels(request); + return ((GetNodeToLabelsResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public GetLabelsResponseProto getLabels(RpcController controller, + GetLabelsRequestProto proto) throws ServiceException { + GetLabelsRequestPBImpl request = + new GetLabelsRequestPBImpl(proto); + try { + GetLabelsResponse response = real.getLabels(request); + return ((GetLabelsResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } + + @Override + public ClearAllLabelsResponseProto clearAllLabels(RpcController controller, + ClearAllLabelsRequestProto proto) throws ServiceException { + ClearAllLabelsRequestPBImpl request = + new ClearAllLabelsRequestPBImpl(proto); + try { + ClearAllLabelsResponse response = real.clearAllLabels(request); + return ((ClearAllLabelsResponsePBImpl) response).getProto(); + } catch (YarnException e) { + throw new ServiceException(e); + } catch (IOException e) { + throw new ServiceException(e); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index ff0a249..43232d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -57,6 +57,14 @@ import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; +import org.apache.hadoop.yarn.server.api.protocolrecords.AddLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.AddLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.ClearAllLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.ClearAllLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodeToLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.GetNodeToLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest; @@ -69,8 +77,14 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveLabelsResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.SetNodeToLabelsRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.SetNodeToLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetLabelsResponsePBImpl; +import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetNodeToLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent; import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider; @@ -199,7 +213,8 @@ void resetLeaderElection() { } private UserGroupInformation checkAccess(String method) throws IOException { - return RMServerUtils.verifyAccess(adminAcl, method, LOG); + return RMServerUtils.verifyAccess(adminAcl, method, + AdminService.class.getName(), LOG); } private UserGroupInformation checkAcls(String method) throws YarnException { @@ -612,4 +627,111 @@ public AccessControlList getAccessControlList() { public Server getServer() { return this.server; } + + @Override + public AddLabelsResponse addLabels(AddLabelsRequest request) + throws YarnException, IOException { + String argName = "addLabels"; + UserGroupInformation user = checkAcls(argName); + + if (!isRMActive()) { + RMAuditLogger.logFailure(user.getShortUserName(), argName, + adminAcl.toString(), "AdminService", + "ResourceManager is not active. Can not add labels."); + throwStandbyException(); + } + + AddLabelsResponse response = + recordFactory.newRecordInstance(AddLabelsResponse.class); + try { + rmContext.getNodeLabelManager().addLabels(request.getLabels()); + RMAuditLogger + .logSuccess(user.getShortUserName(), argName, "AdminService"); + return response; + } catch (IOException ioe) { + LOG.info("Exception add labels", ioe); + RMAuditLogger.logFailure(user.getShortUserName(), argName, + adminAcl.toString(), "AdminService", "Exception add label"); + throw RPCUtil.getRemoteException(ioe); + } + } + + @Override + public RemoveLabelsResponse removeLabels( + RemoveLabelsRequest request) throws YarnException, IOException { + String argName = "removeLabels"; + UserGroupInformation user = checkAcls(argName); + + if (!isRMActive()) { + RMAuditLogger.logFailure(user.getShortUserName(), argName, + adminAcl.toString(), "AdminService", + "ResourceManager is not active. Can not remove labels."); + throwStandbyException(); + } + + RemoveLabelsResponse response = + recordFactory.newRecordInstance(RemoveLabelsResponse.class); + try { + rmContext.getNodeLabelManager().removeLabels(request.getLabels()); + RMAuditLogger + .logSuccess(user.getShortUserName(), argName, "AdminService"); + return response; + } catch (IOException ioe) { + LOG.info("Exception remove labels", ioe); + RMAuditLogger.logFailure(user.getShortUserName(), argName, + adminAcl.toString(), "AdminService", "Exception remove label"); + throw RPCUtil.getRemoteException(ioe); + } + } + + @Override + public SetNodeToLabelsResponse setNodeToLabels( + SetNodeToLabelsRequest request) throws YarnException, IOException { + String argName = "setNodeToLabels"; + UserGroupInformation user = checkAcls(argName); + + if (!isRMActive()) { + RMAuditLogger.logFailure(user.getShortUserName(), argName, + adminAcl.toString(), "AdminService", + "ResourceManager is not active. Can not set node to labels."); + throwStandbyException(); + } + + SetNodeToLabelsResponse response = + recordFactory.newRecordInstance(SetNodeToLabelsResponse.class); + try { + rmContext.getNodeLabelManager().setLabelsOnMultipleNodes( + request.getNodeToLabels()); + RMAuditLogger + .logSuccess(user.getShortUserName(), argName, "AdminService"); + return response; + } catch (IOException ioe) { + LOG.info("Exception set node to labels. ", ioe); + RMAuditLogger.logFailure(user.getShortUserName(), argName, + adminAcl.toString(), "AdminService", + "Exception set node to labels."); + throw RPCUtil.getRemoteException(ioe); + } + } + + @Override + public GetNodeToLabelsResponse getNodeToLabels(GetNodeToLabelsRequest request) + throws YarnException, IOException { + return GetNodeToLabelsResponsePBImpl.newInstance(rmContext + .getNodeLabelManager().getNodesToLabels()); + } + + @Override + public GetLabelsResponse getLabels(GetLabelsRequest request) + throws YarnException, IOException { + return GetLabelsResponsePBImpl.newInstance(rmContext.getNodeLabelManager() + .getLabels()); + } + + @Override + public ClearAllLabelsResponse clearAllLabels(ClearAllLabelsRequest request) + throws YarnException, IOException { + rmContext.getNodeLabelManager().clearAllLabels(); + return ClearAllLabelsResponse.newInstance(); + } }