From 0dc9d12c75475004880793db57ec4e237ae1557a Mon Sep 17 00:00:00 2001 From: Sunil G Date: Fri, 19 May 2017 00:37:08 +0530 Subject: [PATCH] YARN-6585 --- .../AddToClusterNodeLabelsRequest.java | 29 ++++++++++++ .../pb/AddToClusterNodeLabelsRequestPBImpl.java | 39 +++++++++++++++- .../server/resourcemanager/TestRMAdminService.java | 54 ++++++++++++++++++++++ 3 files changed, 120 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java index f2ac395..16a8fd8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java @@ -18,7 +18,9 @@ package org.apache.hadoop.yarn.server.api.protocolrecords; +import java.util.ArrayList; import java.util.List; +import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Unstable; @@ -37,6 +39,15 @@ public static AddToClusterNodeLabelsRequest newInstance(List nodeLabe return request; } + @Deprecated + public static AddToClusterNodeLabelsRequest newInstance(Set nodeLabels) { + List list = new ArrayList<>(); + for (String label : nodeLabels) { + list.add(NodeLabel.newInstance(label)); + } + return newInstance(list); + } + @Public @Unstable public abstract void setNodeLabels(List nodeLabels); @@ -44,4 +55,22 @@ public static AddToClusterNodeLabelsRequest newInstance(List nodeLabe @Public @Unstable public abstract List getNodeLabels(); + + /** + * Set node labels to the response. + * + * @param labels Node labels + * @deprecated Use {@link #setNodeLabelList(List)} instead. + */ + @Deprecated + public abstract void setNodeLabelsList(Set labels); + + /** + * Get node labels of the response. + * + * @return Node labels + * @deprecated Use {@link #getNodeLabelList()} instead. + */ + @Deprecated + public abstract Set getNodeLabelsList(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java index 0f1e400..670d5b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java @@ -19,7 +19,9 @@ package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl; @@ -111,6 +113,13 @@ private void initLocalNodeLabels() { for (NodeLabelProto r : attributesProtoList) { this.updatedNodeLabels.add(convertFromProtoFormat(r)); } + + if (this.updatedNodeLabels.isEmpty()) { + List deprecatedLabelsList = p.getDeprecatedNodeLabelsList(); + for (String l : deprecatedLabelsList) { + this.updatedNodeLabels.add(NodeLabel.newInstance(l)); + } + } } private NodeLabel convertFromProtoFormat(NodeLabelProto p) { @@ -127,7 +136,7 @@ public String toString() { } @Override - public void setNodeLabels(List nodeLabels) { + public synchronized void setNodeLabels(List nodeLabels) { maybeInitBuilder(); this.updatedNodeLabels = new ArrayList<>(); if (nodeLabels == null) { @@ -138,11 +147,37 @@ public void setNodeLabels(List nodeLabels) { } @Override - public List getNodeLabels() { + public synchronized List getNodeLabels() { if (this.updatedNodeLabels != null) { return this.updatedNodeLabels; } initLocalNodeLabels(); return this.updatedNodeLabels; } + + @Override + @Deprecated + public void setNodeLabelsList(Set labels) { + List list = new ArrayList<>(); + for (String s : labels) { + list.add(NodeLabel.newInstance(s)); + } + setNodeLabels(list); + } + + /** + * @deprecated Use {@link #getNodeLabelList()} instead. + */ + @Override + @Deprecated + public synchronized Set getNodeLabelsList() { + Set set = new HashSet<>(); + List labelList = getNodeLabels(); + if (labelList != null) { + for (NodeLabel label : labelList) { + set.add(label.getName()); + } + } + return set; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 8d00ae0..e109452 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -26,7 +26,10 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintWriter; +import java.net.InetSocketAddress; import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -46,12 +49,17 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.security.authorize.ServiceAuthorizationManager; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.records.DecommissionType; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest; @@ -79,6 +87,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.ipc.YarnRPC; + import static org.junit.Assert.assertTrue; public class TestRMAdminService { @@ -1463,4 +1473,48 @@ private void checkBadConfiguration(Configuration conf) { e.getMessage().startsWith(HAUtil.BAD_CONFIG_MESSAGE_PREFIX)); } } + + @Test(timeout = 30000) + public void testAdminAddToClusterNodeLabelsWithDeprecatedAPIs() + throws Exception, YarnException { + configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, + "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"); + + uploadDefaultConfiguration(); + + rm = new MockRM(configuration) { + protected ClientRMService createClientRMService() { + return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, + this.applicationACLsManager, this.queueACLsManager, + this.getRMContext().getRMDelegationTokenSecretManager()); + }; + }; + rm.init(configuration); + rm.start(); + + try { + Set labels = new HashSet<>(); + labels.add("x"); + labels.add("y"); + rm.adminService.addToClusterNodeLabels( + AddToClusterNodeLabelsRequest.newInstance(labels)); + } catch (Exception ex) { + fail("Could not refresh cluster max priority."); + } + + // Create a client. + Configuration conf = new Configuration(); + YarnRPC rpc = YarnRPC.create(conf); + InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress(); + ApplicationClientProtocol client = (ApplicationClientProtocol) rpc + .getProxy(ApplicationClientProtocol.class, rmAddress, conf); + + // Get node labels collection + GetClusterNodeLabelsResponse response = client + .getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance()); + NodeLabel labelX = NodeLabel.newInstance("x"); + NodeLabel labelY = NodeLabel.newInstance("y"); + Assert.assertTrue( + response.getNodeLabelList().containsAll(Arrays.asList(labelX, labelY))); + } } -- 2.10.1 (Apple Git-78)