diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java index 23da1f4..3a2af11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeLabel.java @@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.Records; @Public @@ -28,6 +29,13 @@ public abstract class NodeLabel { @Public @Unstable + public static NodeLabel newInstance(String nodeLabel) { + return newInstance(nodeLabel, + YarnConfiguration.DEFAULT_NODE_LABEL_EXCLUSIVITY); + } + + @Public + @Unstable public static NodeLabel newInstance(String nodeLabel, boolean isExclusive) { NodeLabel request = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 13e9a10..504757a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1731,6 +1731,11 @@ private static void addDeprecatedKeys() { public static final String DEFAULT_NODELABEL_CONFIGURATION_TYPE = CENTALIZED_NODELABEL_CONFIGURATION_TYPE; + + /** + * By default, node label is exclusive or not + */ + public static final boolean DEFAULT_NODE_LABEL_EXCLUSIVITY = true; public YarnConfiguration() { super(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java index 8777e00..36b3e51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java @@ -48,8 +48,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; @@ -136,10 +134,4 @@ public RemoveFromClusterNodeLabelsResponse removeFromClusterNodeLabels( @Idempotent public ReplaceLabelsOnNodeResponse replaceLabelsOnNode( ReplaceLabelsOnNodeRequest request) throws YarnException, IOException; - - @Public - @Evolving - @Idempotent - public UpdateNodeLabelsResponse updateNodeLabels( - UpdateNodeLabelsRequest request) throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java index 68fe995..2464693 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AddToClusterNodeLabelsRequest.java @@ -18,27 +18,33 @@ package org.apache.hadoop.yarn.server.api.protocolrecords; -import java.util.Set; +import java.util.Collection; +import java.util.List; import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.util.Records; @Public -@Evolving +@Unstable public abstract class AddToClusterNodeLabelsRequest { - public static AddToClusterNodeLabelsRequest newInstance(Set labels) { + @Public + @Unstable + public static AddToClusterNodeLabelsRequest newInstance( + Collection NodeLabels) { AddToClusterNodeLabelsRequest request = - Records.newRecord(AddToClusterNodeLabelsRequest.class); - request.setNodeLabels(labels); + Records.newRecord(AddToClusterNodeLabelsRequest.class); + request.setNodeLabels(NodeLabels); return request; } @Public - @Evolving - public abstract void setNodeLabels(Set labels); - + @Unstable + public abstract void setNodeLabels( + Collection NodeLabels); + @Public - @Evolving - public abstract Set getNodeLabels(); + @Unstable + public abstract List getNodeLabels(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsRequest.java deleted file mode 100644 index 44bdc65..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsRequest.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.protocolrecords; - -import java.util.List; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.records.NodeLabel; -import org.apache.hadoop.yarn.util.Records; - -@Public -@Unstable -public abstract class UpdateNodeLabelsRequest { - @Public - @Unstable - public static UpdateNodeLabelsRequest newInstance( - List NodeLabels) { - UpdateNodeLabelsRequest request = - Records.newRecord(UpdateNodeLabelsRequest.class); - request.setNodeLabels(NodeLabels); - return request; - } - - @Public - @Unstable - public abstract void setNodeLabels( - List NodeLabels); - - @Public - @Unstable - public abstract List getNodeLabels(); -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsResponse.java deleted file mode 100644 index eb704c0..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/UpdateNodeLabelsResponse.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.protocolrecords; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.util.Records; - -@Public -@Unstable -public class UpdateNodeLabelsResponse { - - @Private - @Unstable - public static UpdateNodeLabelsResponse newInstance() { - UpdateNodeLabelsResponse response = - Records.newRecord(UpdateNodeLabelsResponse.class); - return response; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto index 6646718..7f54b8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto @@ -42,5 +42,4 @@ service ResourceManagerAdministrationProtocolService { rpc addToClusterNodeLabels(AddToClusterNodeLabelsRequestProto) returns (AddToClusterNodeLabelsResponseProto); rpc removeFromClusterNodeLabels(RemoveFromClusterNodeLabelsRequestProto) returns (RemoveFromClusterNodeLabelsResponseProto); rpc replaceLabelsOnNodes(ReplaceLabelsOnNodeRequestProto) returns (ReplaceLabelsOnNodeResponseProto); - rpc updateNodeLabels(UpdateNodeLabelsRequestProto) returns (UpdateNodeLabelsResponseProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto index 0d5b5c7..8102c85 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto @@ -76,7 +76,7 @@ message UpdateNodeResourceResponseProto { } message AddToClusterNodeLabelsRequestProto { - repeated string nodeLabels = 1; + repeated NodeLabelProto nodeLabels = 1; } message AddToClusterNodeLabelsResponseProto { @@ -97,11 +97,6 @@ message ReplaceLabelsOnNodeResponseProto { } -message UpdateNodeLabelsRequestProto { - repeated NodeLabelProto nodeLabels = 1; -} - - message UpdateNodeLabelsResponseProto { } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java index c04b7fe..b62b091 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java @@ -54,7 +54,7 @@ private void initializeNodeLabels() throws IOException { RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager(); Set labels = new HashSet(); labels.add("x"); - labelsMgr.addToCluserNodeLabels(labels); + labelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(labels); // Setup queue access to node labels distShellTest.conf.set("yarn.scheduler.capacity.root.accessible-node-labels", "x"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java index 3924803..33b77b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java @@ -124,7 +124,7 @@ void printClusterNodeLabels() throws YarnException, IOException { Set nodeLabels = null; if (accessLocal) { nodeLabels = - getNodeLabelManagerInstance(getConf()).getClusterNodeLabels(); + getNodeLabelManagerInstance(getConf()).getClusterNodeLabelNames(); } else { nodeLabels = client.getClusterNodeLabels(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java index 420eeb0..327bea8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java @@ -39,6 +39,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.client.ClientRMProxy; import org.apache.hadoop.yarn.client.RMHAServiceTarget; import org.apache.hadoop.yarn.conf.HAUtil; @@ -62,6 +63,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; @Private @Unstable @@ -75,6 +77,9 @@ "No cluster node-labels are specified"; private static final String NO_MAPPING_ERR_MSG = "No node-to-labels mappings are specified"; + private static final String ADD_LABEL_FORMAT_ERR_MSG = + "Add cluster node-labels format is not correct, it should be" + + " labelName1[(exclusive=true/false)],LabelName2"; protected final static Map ADMIN_USAGE = ImmutableMap.builder() @@ -96,7 +101,7 @@ .put("-getGroups", new UsageInfo("[username]", "Get the groups which given user belongs to.")) .put("-addToClusterNodeLabels", - new UsageInfo("[label1,label2,label3] (label splitted by \",\")", + new UsageInfo("[label1,label2,label3[(exclusive=true/false)]]", "add to cluster node labels ")) .put("-removeFromClusterNodeLabels", new UsageInfo("[label1,label2,label3] (label splitted by \",\")", @@ -351,7 +356,65 @@ private int getGroups(String[] usernames) throws IOException { return localNodeLabelsManager; } - private Set buildNodeLabelsSetFromStr(String args) { + private List buildNodeLabelsFromStr(String args) { + List nodeLabels = new ArrayList<>(); + for (String p : args.split(",")) { + if (!p.trim().isEmpty()) { + String labelName = p; + + // Try to parse exclusive + boolean exclusive = YarnConfiguration.DEFAULT_NODE_LABEL_EXCLUSIVITY; + int leftParenthesisIdx = p.indexOf("("); + int rightParenthesisIdx = p.indexOf(")"); + + if ((leftParenthesisIdx == -1 && rightParenthesisIdx != -1) + || (leftParenthesisIdx != -1 && rightParenthesisIdx == -1)) { + // Parenthese not match + throw new IllegalArgumentException(ADD_LABEL_FORMAT_ERR_MSG); + } + + if (leftParenthesisIdx > 0 && rightParenthesisIdx > 0) { + if (leftParenthesisIdx > rightParenthesisIdx) { + // Parentese not match + throw new IllegalArgumentException(ADD_LABEL_FORMAT_ERR_MSG); + } + + String property = p.substring(p.indexOf("(") + 1, p.indexOf(")")); + if (property.contains("=")) { + String key = property.substring(0, property.indexOf("=")).trim(); + String value = + property.substring(property.indexOf("=") + 1, property.length()) + .trim(); + + // Now we only support one property, which is exclusive, so check if + // key = exclusive and value = {true/false} + if (key.equals("exclusive") + && ImmutableSet.of("true", "false").contains(value)) { + exclusive = Boolean.parseBoolean(value); + } else { + throw new IllegalArgumentException(ADD_LABEL_FORMAT_ERR_MSG); + } + } else { + throw new IllegalArgumentException(ADD_LABEL_FORMAT_ERR_MSG); + } + } + + // Try to get labelName if there's "(..)" + if (labelName.contains("(")) { + labelName = labelName.substring(0, labelName.indexOf("(")).trim(); + } + + nodeLabels.add(NodeLabel.newInstance(labelName, exclusive)); + } + } + + if (nodeLabels.isEmpty()) { + throw new IllegalArgumentException(NO_LABEL_ERR_MSG); + } + return nodeLabels; + } + + private Set buildNodeLabelNamesFromStr(String args) { Set labels = new HashSet(); for (String p : args.split(",")) { if (!p.trim().isEmpty()) { @@ -367,7 +430,7 @@ private int getGroups(String[] usernames) throws IOException { private int addToClusterNodeLabels(String args) throws IOException, YarnException { - Set labels = buildNodeLabelsSetFromStr(args); + List labels = buildNodeLabelsFromStr(args); if (directlyAccessNodeLabelStore) { getNodeLabelManagerInstance(getConf()).addToCluserNodeLabels(labels); @@ -383,7 +446,7 @@ private int addToClusterNodeLabels(String args) throws IOException, private int removeFromClusterNodeLabels(String args) throws IOException, YarnException { - Set labels = buildNodeLabelsSetFromStr(args); + Set labels = buildNodeLabelNamesFromStr(args); if (directlyAccessNodeLabelStore) { getNodeLabelManagerInstance(getConf()).removeFromClusterNodeLabels( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java index f9ccf87..1153b10 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java @@ -82,7 +82,7 @@ public void testGetClusterNodeLabelsWithLocalAccess() throws Exception { cli.setSysOutPrintStream(sysOut); cli.setSysErrPrintStream(sysErr); ClusterCLI.localNodeLabelsManager = mock(CommonNodeLabelsManager.class); - when(ClusterCLI.localNodeLabelsManager.getClusterNodeLabels()) + when(ClusterCLI.localNodeLabelsManager.getClusterNodeLabelNames()) .thenReturn(ImmutableSet.of("local1", "local2")); int rc = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java index 6067110..0827bdd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java @@ -18,7 +18,9 @@ package org.apache.hadoop.yarn.client.cli; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.argThat; @@ -38,7 +40,6 @@ import org.apache.hadoop.ha.HAServiceTarget; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.client.cli.RMAdminCLI; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; @@ -408,7 +409,7 @@ public void testAccessLocalNodeLabelManager() throws Exception { String[] args = { "-addToClusterNodeLabels", "x,y", "-directlyAccessNodeLabelStore" }; assertEquals(0, rmAdminCLI.run(args)); - assertTrue(dummyNodeLabelsManager.getClusterNodeLabels().containsAll( + assertTrue(dummyNodeLabelsManager.getClusterNodeLabelNames().containsAll( ImmutableSet.of("x", "y"))); // reset localNodeLabelsManager @@ -420,7 +421,7 @@ public void testAccessLocalNodeLabelManager() throws Exception { new String[] { "-addToClusterNodeLabels", "-directlyAccessNodeLabelStore", "x,y" }; assertEquals(0, rmAdminCLI.run(args)); - assertTrue(dummyNodeLabelsManager.getClusterNodeLabels().containsAll( + assertTrue(dummyNodeLabelsManager.getClusterNodeLabelNames().containsAll( ImmutableSet.of("x", "y"))); // local node labels manager will be close after running @@ -434,7 +435,7 @@ public void testAccessRemoteNodeLabelManager() throws Exception { assertEquals(0, rmAdminCLI.run(args)); // localNodeLabelsManager shouldn't accessed - assertTrue(dummyNodeLabelsManager.getClusterNodeLabels().isEmpty()); + assertTrue(dummyNodeLabelsManager.getClusterNodeLabelNames().isEmpty()); // remote node labels manager accessed assertTrue(remoteAdminServiceAccessed); @@ -446,7 +447,7 @@ public void testAddToClusterNodeLabels() throws Exception { String[] args = { "-addToClusterNodeLabels", "x", "-directlyAccessNodeLabelStore" }; assertEquals(0, rmAdminCLI.run(args)); - assertTrue(dummyNodeLabelsManager.getClusterNodeLabels().containsAll( + assertTrue(dummyNodeLabelsManager.getClusterNodeLabelNames().containsAll( ImmutableSet.of("x"))); // no labels, should fail @@ -472,19 +473,62 @@ public void testAddToClusterNodeLabels() throws Exception { new String[] { "-addToClusterNodeLabels", ",x,,", "-directlyAccessNodeLabelStore" }; assertEquals(0, rmAdminCLI.run(args)); - assertTrue(dummyNodeLabelsManager.getClusterNodeLabels().containsAll( + assertTrue(dummyNodeLabelsManager.getClusterNodeLabelNames().containsAll( ImmutableSet.of("x"))); } @Test + public void testAddToClusterNodeLabelsWithExclusivitySetting() + throws Exception { + // Parenthese not match + String[] args = new String[] { "-addToClusterNodeLabels", "x(" }; + assertTrue(0 != rmAdminCLI.run(args)); + + args = new String[] { "-addToClusterNodeLabels", "x)" }; + assertTrue(0 != rmAdminCLI.run(args)); + + // No key=value specifying inner parenthese + args = new String[] { "-addToClusterNodeLabels", "x()" }; + assertTrue(0 != rmAdminCLI.run(args)); + + // Not expected key=value specifying inner parentese + args = new String[] { "-addToClusterNodeLabels", "x(key=value)" }; + assertTrue(0 != rmAdminCLI.run(args)); + + // Not key is expected, but value not + args = new String[] { "-addToClusterNodeLabels", "x(exclusive=)" }; + assertTrue(0 != rmAdminCLI.run(args)); + + // key=value both set + args = + new String[] { "-addToClusterNodeLabels", + "x(exclusive=true), y(exclusive=false)", + "-directlyAccessNodeLabelStore" }; + assertTrue(0 == rmAdminCLI.run(args)); + + assertTrue(dummyNodeLabelsManager.isExclusiveNodeLabel("x")); + assertFalse(dummyNodeLabelsManager.isExclusiveNodeLabel("y")); + + // key=value both set, and some spaces need to be handled + args = + new String[] { "-addToClusterNodeLabels", + "a (exclusive= true) , b( exclusive =false)", + "-directlyAccessNodeLabelStore" }; + assertTrue(0 == rmAdminCLI.run(args)); + + assertTrue(dummyNodeLabelsManager.isExclusiveNodeLabel("a")); + assertFalse(dummyNodeLabelsManager.isExclusiveNodeLabel("b")); + } + + @Test public void testRemoveFromClusterNodeLabels() throws Exception { // Successfully remove labels - dummyNodeLabelsManager.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + dummyNodeLabelsManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); String[] args = { "-removeFromClusterNodeLabels", "x,,y", "-directlyAccessNodeLabelStore" }; assertEquals(0, rmAdminCLI.run(args)); - assertTrue(dummyNodeLabelsManager.getClusterNodeLabels().isEmpty()); + assertTrue(dummyNodeLabelsManager.getClusterNodeLabelNames().isEmpty()); // no labels, should fail args = new String[] { "-removeFromClusterNodeLabels" }; @@ -509,7 +553,7 @@ public void testRemoveFromClusterNodeLabels() throws Exception { public void testReplaceLabelsOnNode() throws Exception { // Successfully replace labels dummyNodeLabelsManager - .addToCluserNodeLabels(ImmutableSet.of("x", "y", "Y")); + .addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "Y")); String[] args = { "-replaceLabelsOnNode", "node1:8000,x node2:8000=y node3,x node4=Y", @@ -544,7 +588,7 @@ public void testReplaceLabelsOnNode() throws Exception { @Test public void testReplaceMultipleLabelsOnSingleNode() throws Exception { // Successfully replace labels - dummyNodeLabelsManager.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + dummyNodeLabelsManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); String[] args = { "-replaceLabelsOnNode", "node1,x,y", "-directlyAccessNodeLabelStore" }; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index fe38164..44d47f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.nodelabels; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; @@ -53,7 +54,6 @@ import org.apache.hadoop.yarn.nodelabels.event.NodeLabelsStoreEventType; import org.apache.hadoop.yarn.nodelabels.event.RemoveClusterNodeLabels; import org.apache.hadoop.yarn.nodelabels.event.StoreNewClusterNodeLabels; -import org.apache.hadoop.yarn.nodelabels.event.StoreUpdateNodeLabelsEvent; import org.apache.hadoop.yarn.nodelabels.event.UpdateNodeToLabelsMappingsEvent; import org.apache.hadoop.yarn.util.resource.Resources; @@ -185,13 +185,6 @@ protected void handleStoreEvent(NodeLabelsStoreEvent event) { store.updateNodeToLabelsMappings(updateNodeToLabelsMappingsEvent .getNodeToLabels()); break; - case UPDATE_NODE_LABELS: - StoreUpdateNodeLabelsEvent - storeSetNodeLabelsEventEvent = - (StoreUpdateNodeLabelsEvent) event; - store.updateNodeLabels(storeSetNodeLabelsEventEvent - .getUpdatedNodeLabels()); - break; } } catch (IOException e) { LOG.error("Failed to store label modification to storage"); @@ -273,15 +266,10 @@ protected void serviceStop() throws Exception { store.close(); } } - - /** - * Add multiple node labels to repository - * - * @param labels - * new node labels added - */ + @SuppressWarnings("unchecked") - public void addToCluserNodeLabels(Set labels) throws IOException { + public void addToCluserNodeLabels(Collection labels) + throws IOException { if (!nodeLabelsEnabled) { LOG.error(NODE_LABELS_NOT_ENABLED_ERR); throw new IOException(NODE_LABELS_NOT_ENABLED_ERR); @@ -289,19 +277,19 @@ public void addToCluserNodeLabels(Set labels) throws IOException { if (null == labels || labels.isEmpty()) { return; } - Set newLabels = new HashSet(); - labels = normalizeLabels(labels); + List newLabels = new ArrayList(); + normalizeNodeLabels(labels); // do a check before actual adding them, will throw exception if any of them // doesn't meet label name requirement - for (String label : labels) { - checkAndThrowLabelName(label); + for (NodeLabel label : labels) { + checkAndThrowLabelName(label.getNodeLabel()); } - for (String label : labels) { + for (NodeLabel label : labels) { // shouldn't overwrite it to avoid changing the Label.resource - if (this.labelCollections.get(label) == null) { - this.labelCollections.put(label, new RMNodeLabel(label)); + if (this.labelCollections.get(label.getNodeLabel()) == null) { + this.labelCollections.put(label.getNodeLabel(), new RMNodeLabel(label)); newLabels.add(label); } } @@ -310,7 +298,23 @@ public void addToCluserNodeLabels(Set labels) throws IOException { new StoreNewClusterNodeLabels(newLabels)); } - LOG.info("Add labels: [" + StringUtils.join(labels.iterator(), ",") + "]"); + LOG.info("Add labels: [" + StringUtils.join(labels.iterator(), ",") + "]"); + } + + /** + * Add multiple node labels to repository + * + * @param labels + * new node labels added + */ + @VisibleForTesting + public void addToCluserNodeLabelsWithDefaultExclusivity(Set labels) + throws IOException { + Set nodeLabels = new HashSet(); + for (String label : labels) { + nodeLabels.add(NodeLabel.newInstance(label)); + } + addToCluserNodeLabels(nodeLabels); } protected void checkAddLabelsToNode( @@ -780,7 +784,7 @@ public void replaceLabelsOnNode(Map> replaceLabelsToNode) * * @return existing valid labels in repository */ - public Set getClusterNodeLabels() { + public Set getClusterNodeLabelNames() { try { readLock.lock(); Set labels = new HashSet(labelCollections.keySet()); @@ -791,39 +795,17 @@ public void replaceLabelsOnNode(Map> replaceLabelsToNode) } } - private void checkUpdateNodeLabels( - List updatedNodeLabels) throws YarnException { - // pre-check - for (NodeLabel label : updatedNodeLabels) { - if (!labelCollections.containsKey(label.getNodeLabel())) { - String message = - String.format( - "Trying to update a non-existing node-label=%s", - label.getNodeLabel()); - LOG.error(message); - throw new YarnException(message); - } - } - } - - @SuppressWarnings("unchecked") - public void updateNodeLabels( - List updatedNodeLabels) throws YarnException { + public List getClusterNodeLabels() { try { - writeLock.lock(); - checkUpdateNodeLabels(updatedNodeLabels); - - for (NodeLabel label : updatedNodeLabels) { - RMNodeLabel rmLabel = labelCollections.get(label.getNodeLabel()); - rmLabel.setIsExclusive(label.getIsExclusive()); - } - - if (null != dispatcher && !updatedNodeLabels.isEmpty()) { - dispatcher.getEventHandler().handle( - new StoreUpdateNodeLabelsEvent(updatedNodeLabels)); + readLock.lock(); + List nodeLabels = new ArrayList<>(); + for (RMNodeLabel label : labelCollections.values()) { + nodeLabels.add(NodeLabel.newInstance(label.getLabelName(), + label.getIsExclusive())); } + return nodeLabels; } finally { - writeLock.unlock(); + readLock.unlock(); } } @@ -876,6 +858,12 @@ protected String normalizeLabel(String label) { return newLabels; } + private void normalizeNodeLabels(Collection labels) { + for (NodeLabel label : labels) { + label.setNodeLabel(normalizeLabel(label.getNodeLabel())); + } + } + protected Node getNMInNodeSet(NodeId nodeId) { return getNMInNodeSet(nodeId, nodeCollections); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java index 0c7192f..58684df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java @@ -38,18 +38,16 @@ import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RemoveFromClusterNodeLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsRequestProto; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RemoveFromClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeRequestPBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeLabelsRequestPBImpl; import com.google.common.collect.Sets; @@ -66,7 +64,7 @@ public FileSystemNodeLabelsStore(CommonNodeLabelsManager mgr) { protected static final String EDITLOG_FILENAME = "nodelabel.editlog"; protected enum SerializedLogType { - ADD_LABELS, NODE_TO_LABELS, REMOVE_LABELS, UPDATE_NODE_LABELS + ADD_LABELS, NODE_TO_LABELS, REMOVE_LABELS } Path fsWorkingPath; @@ -138,12 +136,12 @@ public void updateNodeToLabelsMappings( } @Override - public void storeNewClusterNodeLabels(Set labels) + public void storeNewClusterNodeLabels(List labels) throws IOException { ensureAppendEditlogFile(); editlogOs.writeInt(SerializedLogType.ADD_LABELS.ordinal()); - ((AddToClusterNodeLabelsRequestPBImpl) AddToClusterNodeLabelsRequest.newInstance(labels)).getProto() - .writeDelimitedTo(editlogOs); + ((AddToClusterNodeLabelsRequestPBImpl) AddToClusterNodeLabelsRequest + .newInstance(labels)).getProto().writeDelimitedTo(editlogOs); ensureCloseEditlogFile(); } @@ -156,16 +154,6 @@ public void removeClusterNodeLabels(Collection labels) .newHashSet(labels.iterator()))).getProto().writeDelimitedTo(editlogOs); ensureCloseEditlogFile(); } - - @Override - public void updateNodeLabels(List updatedNodeLabels) - throws IOException { - ensureAppendEditlogFile(); - editlogOs.writeInt(SerializedLogType.UPDATE_NODE_LABELS.ordinal()); - ((UpdateNodeLabelsRequestPBImpl) UpdateNodeLabelsRequest - .newInstance(updatedNodeLabels)).getProto().writeDelimitedTo(editlogOs); - ensureCloseEditlogFile(); - } @Override public void recover() throws YarnException, IOException { @@ -192,7 +180,7 @@ public void recover() throws YarnException, IOException { } if (null != is) { - Set labels = + List labels = new AddToClusterNodeLabelsRequestPBImpl( AddToClusterNodeLabelsRequestProto.parseDelimitedFrom(is)).getNodeLabels(); Map> nodeToLabels = @@ -216,10 +204,11 @@ public void recover() throws YarnException, IOException { switch (type) { case ADD_LABELS: { - Collection labels = - AddToClusterNodeLabelsRequestProto.parseDelimitedFrom(is) - .getNodeLabelsList(); - mgr.addToCluserNodeLabels(Sets.newHashSet(labels.iterator())); + List labels = + new AddToClusterNodeLabelsRequestPBImpl( + AddToClusterNodeLabelsRequestProto.parseDelimitedFrom(is)) + .getNodeLabels(); + mgr.addToCluserNodeLabels(labels); break; } case REMOVE_LABELS: { @@ -237,14 +226,6 @@ public void recover() throws YarnException, IOException { mgr.replaceLabelsOnNode(map); break; } - case UPDATE_NODE_LABELS: { - List attributes = - new UpdateNodeLabelsRequestPBImpl( - UpdateNodeLabelsRequestProto.parseDelimitedFrom(is)) - .getNodeLabels(); - mgr.updateNodeLabels(attributes); - break; - } } } catch (EOFException e) { // EOF hit, break diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java index 6bd90db..47b7370 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java @@ -46,7 +46,7 @@ public abstract void updateNodeToLabelsMappings( /** * Store new labels */ - public abstract void storeNewClusterNodeLabels(Set label) + public abstract void storeNewClusterNodeLabels(List label) throws IOException; /** @@ -54,13 +54,7 @@ public abstract void storeNewClusterNodeLabels(Set label) */ public abstract void removeClusterNodeLabels(Collection labels) throws IOException; - - /** - * Update node labels - */ - public abstract void updateNodeLabels( - List updatedNodeLabels) throws IOException; - + /** * Recover labels and node to labels mappings from store */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java index 7638e21..b0ab39b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java @@ -20,9 +20,12 @@ import java.util.HashSet; import java.util.Set; + import org.apache.commons.lang.StringUtils; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.resource.Resources; public class RMNodeLabel implements Comparable { @@ -30,17 +33,25 @@ private int numActiveNMs; private String labelName; private Set nodeIds; - private boolean exclusive = true; - + private boolean exclusive; + + public RMNodeLabel(NodeLabel nodeLabel) { + this(nodeLabel.getNodeLabel(), Resource.newInstance(0, 0), 0, + nodeLabel.getIsExclusive()); + } + public RMNodeLabel(String labelName) { - this(labelName, Resource.newInstance(0, 0), 0); + this(labelName, Resource.newInstance(0, 0), 0, + YarnConfiguration.DEFAULT_NODE_LABEL_EXCLUSIVITY); } - protected RMNodeLabel(String labelName, Resource res, int activeNMs) { + protected RMNodeLabel(String labelName, Resource res, int activeNMs, + boolean exclusive) { this.labelName = labelName; this.resource = res; this.numActiveNMs = activeNMs; this.nodeIds = new HashSet(); + this.exclusive = exclusive; } public void addNodeId(NodeId node) { @@ -86,7 +97,7 @@ public boolean getIsExclusive() { } public RMNodeLabel getCopy() { - return new RMNodeLabel(labelName, resource, numActiveNMs); + return new RMNodeLabel(labelName, resource, numActiveNMs, exclusive); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/NodeLabelsStoreEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/NodeLabelsStoreEventType.java index ce29176..efa2dbe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/NodeLabelsStoreEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/NodeLabelsStoreEventType.java @@ -21,6 +21,5 @@ public enum NodeLabelsStoreEventType { REMOVE_LABELS, ADD_LABELS, - STORE_NODE_TO_LABELS, - UPDATE_NODE_LABELS + STORE_NODE_TO_LABELS } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/StoreNewClusterNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/StoreNewClusterNodeLabels.java index b478c6b..c5a7d01 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/StoreNewClusterNodeLabels.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/StoreNewClusterNodeLabels.java @@ -18,17 +18,19 @@ package org.apache.hadoop.yarn.nodelabels.event; -import java.util.Set; +import java.util.List; + +import org.apache.hadoop.yarn.api.records.NodeLabel; public class StoreNewClusterNodeLabels extends NodeLabelsStoreEvent { - private Set labels; + private List labels; - public StoreNewClusterNodeLabels(Set labels) { + public StoreNewClusterNodeLabels(List labels) { super(NodeLabelsStoreEventType.ADD_LABELS); this.labels = labels; } - public Set getLabels() { + public List getLabels() { return labels; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/StoreUpdateNodeLabelsEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/StoreUpdateNodeLabelsEvent.java deleted file mode 100644 index 7c05012..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/event/StoreUpdateNodeLabelsEvent.java +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.nodelabels.event; - -import java.util.List; - -import org.apache.hadoop.yarn.api.records.NodeLabel; - -public class StoreUpdateNodeLabelsEvent extends NodeLabelsStoreEvent { - private List updatedNodeLabels; - - public StoreUpdateNodeLabelsEvent(List updateNodeLabels) { - super(NodeLabelsStoreEventType.UPDATE_NODE_LABELS); - this.updatedNodeLabels = updateNodeLabels; - } - - public List getUpdatedNodeLabels() { - return updatedNodeLabels; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java index 8cb225f..c2d813b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java @@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshUserToGroupsMappingsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RemoveFromClusterNodeLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceRequestProto; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB; @@ -62,8 +61,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl; @@ -84,8 +81,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RemoveFromClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeResponsePBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeLabelsRequestPBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceResponsePBImpl; @@ -268,18 +263,4 @@ public ReplaceLabelsOnNodeResponse replaceLabelsOnNode( return null; } } - - @Override - public UpdateNodeLabelsResponse updateNodeLabels( - UpdateNodeLabelsRequest request) throws YarnException, IOException { - UpdateNodeLabelsRequestProto requestProto = - ((UpdateNodeLabelsRequestPBImpl) request).getProto(); - try { - return new UpdateNodeLabelsResponsePBImpl( - proxy.updateNodeLabels(null, requestProto)); - } catch (ServiceException e) { - RPCUtil.unwrapAndThrowException(e); - return null; - } - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java index fe4c812..0eaf581 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java @@ -42,8 +42,6 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RemoveFromClusterNodeLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeResponseProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsRequestProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceResponseProto; import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol; @@ -57,8 +55,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshUserToGroupsMappingsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl; @@ -78,8 +74,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RemoveFromClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeResponsePBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeLabelsRequestPBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceResponsePBImpl; @@ -274,21 +268,4 @@ public ReplaceLabelsOnNodeResponseProto replaceLabelsOnNodes( throw new ServiceException(e); } } - - @Override - public UpdateNodeLabelsResponseProto updateNodeLabels( - RpcController controller, UpdateNodeLabelsRequestProto proto) - throws ServiceException { - UpdateNodeLabelsRequest request = - new UpdateNodeLabelsRequestPBImpl(proto); - try { - UpdateNodeLabelsResponse response = - real.updateNodeLabels(request); - return ((UpdateNodeLabelsResponsePBImpl) response).getProto(); - } catch (YarnException e) { - throw new ServiceException(e); - } catch (IOException e) { - throw new ServiceException(e); - } - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java index 7bf92af..4f3edf9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java @@ -18,23 +18,27 @@ package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; -import java.util.HashSet; -import java.util.Set; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import org.apache.hadoop.yarn.api.records.NodeLabel; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProtoOrBuilder; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; public class AddToClusterNodeLabelsRequestPBImpl extends AddToClusterNodeLabelsRequest { - Set labels; - AddToClusterNodeLabelsRequestProto proto = AddToClusterNodeLabelsRequestProto - .getDefaultInstance(); + AddToClusterNodeLabelsRequestProto proto = + AddToClusterNodeLabelsRequestProto.getDefaultInstance(); AddToClusterNodeLabelsRequestProto.Builder builder = null; + private List updatedNodeLabels; boolean viaProto = false; public AddToClusterNodeLabelsRequestPBImpl() { - this.builder = AddToClusterNodeLabelsRequestProto.newBuilder(); + builder = AddToClusterNodeLabelsRequestProto.newBuilder(); } public AddToClusterNodeLabelsRequestPBImpl( @@ -43,18 +47,11 @@ public AddToClusterNodeLabelsRequestPBImpl( viaProto = true; } - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = AddToClusterNodeLabelsRequestProto.newBuilder(proto); - } - viaProto = false; - } - - private void mergeLocalToBuilder() { - if (this.labels != null && !this.labels.isEmpty()) { - builder.clearNodeLabels(); - builder.addAllNodeLabels(this.labels); - } + public AddToClusterNodeLabelsRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; } private void mergeLocalToProto() { @@ -65,35 +62,31 @@ private void mergeLocalToProto() { viaProto = true; } - public AddToClusterNodeLabelsRequestProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - private void initLabels() { - if (this.labels != null) { - return; + private void mergeLocalToBuilder() { + if (this.updatedNodeLabels != null) { + addNodeLabelsToProto(); } - AddToClusterNodeLabelsRequestProtoOrBuilder p = viaProto ? proto : builder; - this.labels = new HashSet(); - this.labels.addAll(p.getNodeLabelsList()); } - @Override - public void setNodeLabels(Set labels) { + private void addNodeLabelsToProto() { maybeInitBuilder(); - if (labels == null || labels.isEmpty()) { - builder.clearNodeLabels(); + builder.clearNodeLabels(); + List protoList = + new ArrayList(); + for (NodeLabel r : this.updatedNodeLabels) { + protoList.add(convertToProtoFormat(r)); } - this.labels = labels; + builder.addAllNodeLabels(protoList); } @Override - public Set getNodeLabels() { - initLabels(); - return this.labels; + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; } @Override @@ -101,14 +94,53 @@ public int hashCode() { assert false : "hashCode not designed"; return 0; } - + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = AddToClusterNodeLabelsRequestProto.newBuilder(proto); + } + viaProto = false; + } + @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); + public void setNodeLabels(Collection updatedNodeLabels) { + maybeInitBuilder(); + if (updatedNodeLabels == null) { + builder.clearNodeLabels(); } - return false; + this.updatedNodeLabels = new ArrayList<>(updatedNodeLabels); + } + + private void initLocalNodeLabels() { + AddToClusterNodeLabelsRequestProtoOrBuilder p = viaProto ? proto : builder; + List attributesProtoList = + p.getNodeLabelsList(); + this.updatedNodeLabels = new ArrayList(); + for (NodeLabelProto r : attributesProtoList) { + this.updatedNodeLabels.add(convertFromProtoFormat(r)); + } + } + + @Override + public List getNodeLabels() { + if (this.updatedNodeLabels != null) { + return this.updatedNodeLabels; + } + initLocalNodeLabels(); + return this.updatedNodeLabels; + } + + private NodeLabel + convertFromProtoFormat(NodeLabelProto p) { + return new NodeLabelPBImpl(p); + } + + private NodeLabelProto convertToProtoFormat(NodeLabel t) { + return ((NodeLabelPBImpl) t).getProto(); + } + + @Override + public String toString() { + return getProto().toString(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeLabelsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeLabelsRequestPBImpl.java deleted file mode 100644 index f18a8db..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeLabelsRequestPBImpl.java +++ /dev/null @@ -1,145 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.yarn.api.records.NodeLabel; -import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsRequestProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsRequestProtoOrBuilder; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsRequest; - -public class UpdateNodeLabelsRequestPBImpl extends - UpdateNodeLabelsRequest { - UpdateNodeLabelsRequestProto proto = - UpdateNodeLabelsRequestProto.getDefaultInstance(); - UpdateNodeLabelsRequestProto.Builder builder = null; - private List updatedNodeLabels; - boolean viaProto = false; - - public UpdateNodeLabelsRequestPBImpl() { - builder = UpdateNodeLabelsRequestProto.newBuilder(); - } - - public UpdateNodeLabelsRequestPBImpl( - UpdateNodeLabelsRequestProto proto) { - this.proto = proto; - viaProto = true; - } - - public UpdateNodeLabelsRequestProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - private void mergeLocalToProto() { - if (viaProto) - maybeInitBuilder(); - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void mergeLocalToBuilder() { - if (this.updatedNodeLabels != null) { - addNodeLabelsToProto(); - } - } - - private void addNodeLabelsToProto() { - maybeInitBuilder(); - builder.clearNodeLabels(); - List protoList = - new ArrayList(); - for (NodeLabel r : this.updatedNodeLabels) { - protoList.add(convertToProtoFormat(r)); - } - builder.addAllNodeLabels(protoList); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public int hashCode() { - assert false : "hashCode not designed"; - return 0; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = UpdateNodeLabelsRequestProto.newBuilder(proto); - } - viaProto = false; - } - - @Override - public void setNodeLabels(List updatedNodeLabels) { - maybeInitBuilder(); - if (updatedNodeLabels == null) { - builder.clearNodeLabels(); - } - this.updatedNodeLabels = updatedNodeLabels; - } - - private void initLocalNodeLabels() { - UpdateNodeLabelsRequestProtoOrBuilder p = viaProto ? proto : builder; - List attributesProtoList = - p.getNodeLabelsList(); - this.updatedNodeLabels = new ArrayList(); - for (NodeLabelProto r : attributesProtoList) { - this.updatedNodeLabels.add(convertFromProtoFormat(r)); - } - } - - @Override - public List getNodeLabels() { - if (this.updatedNodeLabels != null) { - return this.updatedNodeLabels; - } - initLocalNodeLabels(); - return this.updatedNodeLabels; - } - - private NodeLabel - convertFromProtoFormat(NodeLabelProto p) { - return new NodeLabelPBImpl(p); - } - - private NodeLabelProto convertToProtoFormat(NodeLabel t) { - return ((NodeLabelPBImpl) t).getProto(); - } - - @Override - public String toString() { - return getProto().toString(); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeLabelsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeLabelsResponsePBImpl.java deleted file mode 100644 index 217b8fc..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeLabelsResponsePBImpl.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; - -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsResponseProto; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsResponse; - -public class UpdateNodeLabelsResponsePBImpl extends - UpdateNodeLabelsResponse { - UpdateNodeLabelsResponseProto proto = - UpdateNodeLabelsResponseProto.getDefaultInstance(); - UpdateNodeLabelsResponseProto.Builder builder = null; - boolean viaProto = false; - - public UpdateNodeLabelsResponsePBImpl() { - builder = UpdateNodeLabelsResponseProto.newBuilder(); - } - - public UpdateNodeLabelsResponsePBImpl( - UpdateNodeLabelsResponseProto proto) { - this.proto = proto; - viaProto = true; - } - - public UpdateNodeLabelsResponseProto getProto() { - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return getProto().toString().replaceAll("\\n", ", ") - .replaceAll("\\s+", " "); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 80299c0..d136598 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -230,8 +230,6 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RemoveFromClusterNodeLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeResponseProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsRequestProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateRequestProto; @@ -306,8 +304,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RemoveFromClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeResponsePBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeLabelsRequestPBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceResponsePBImpl; import org.apache.hadoop.yarn.util.resource.Resources; @@ -1287,16 +1283,4 @@ public void testNodeLabelAttributesPBImpl() throws Exception { validatePBImplRecord(NodeLabelPBImpl.class, NodeLabelProto.class); } - - @Test - public void testUpdateNodeLabelsRequestPBImpl() throws Exception { - validatePBImplRecord(UpdateNodeLabelsRequestPBImpl.class, - UpdateNodeLabelsRequestProto.class); - } - - @Test - public void testUpdateNodeLabelsResponsePBImpl() throws Exception { - validatePBImplRecord(UpdateNodeLabelsResponsePBImpl.class, - UpdateNodeLabelsResponseProto.class); - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java index 67e6119..48d6dc8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java @@ -31,9 +31,8 @@ public class DummyCommonNodeLabelsManager extends CommonNodeLabelsManager { Map> lastNodeToLabels = null; - Collection lastAddedlabels = null; + Collection lastAddedlabels = null; Collection lastRemovedlabels = null; - List lastUpdatedNodeLabels = null; @Override public void initNodeLabelStore(Configuration conf) { @@ -56,17 +55,11 @@ public void updateNodeToLabelsMappings( } @Override - public void storeNewClusterNodeLabels(Set label) throws IOException { + public void storeNewClusterNodeLabels(List label) throws IOException { lastAddedlabels = label; } @Override - public void updateNodeLabels(List updatedNodeLabels) - throws IOException { - lastUpdatedNodeLabels = updatedNodeLabels; - } - - @Override public void close() throws IOException { // do nothing } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java index 3b9825d..4d406fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java @@ -88,6 +88,7 @@ public static void assertCollectionEquals(Collection c1, Assert.assertTrue(s1.containsAll(s2)); } + @SuppressWarnings("unchecked") public static Set toSet(E... elements) { Set set = Sets.newHashSet(elements); return set; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java index 1e2326b..86b59a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Collection; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -31,7 +32,6 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -61,15 +61,14 @@ public void after() { @Test(timeout = 5000) public void testAddRemovelabel() throws Exception { // Add some label - mgr.addToCluserNodeLabels(ImmutableSet.of("hello")); - assertCollectionEquals(mgr.lastAddedlabels, Arrays.asList("hello")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("hello")); + verifyNodeLabelAdded(Sets.newHashSet("hello"), mgr.lastAddedlabels); - mgr.addToCluserNodeLabels(ImmutableSet.of("world")); - mgr.addToCluserNodeLabels(toSet("hello1", "world1")); - assertCollectionEquals(mgr.lastAddedlabels, - Sets.newHashSet("hello1", "world1")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("world")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("hello1", "world1")); + verifyNodeLabelAdded(Sets.newHashSet("hello1", "world1"), mgr.lastAddedlabels); - Assert.assertTrue(mgr.getClusterNodeLabels().containsAll( + Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll( Sets.newHashSet("hello", "world", "hello1", "world1"))); // try to remove null, empty and non-existed label, should fail @@ -86,23 +85,32 @@ public void testAddRemovelabel() throws Exception { // Remove some label mgr.removeFromClusterNodeLabels(Arrays.asList("hello")); - assertCollectionEquals(mgr.lastRemovedlabels, Arrays.asList("hello")); - Assert.assertTrue(mgr.getClusterNodeLabels().containsAll( + assertCollectionEquals(Sets.newHashSet("hello"), mgr.lastRemovedlabels); + Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll( Arrays.asList("world", "hello1", "world1"))); mgr.removeFromClusterNodeLabels(Arrays .asList("hello1", "world1", "world")); Assert.assertTrue(mgr.lastRemovedlabels.containsAll(Sets.newHashSet( "hello1", "world1", "world"))); - Assert.assertTrue(mgr.getClusterNodeLabels().isEmpty()); + Assert.assertTrue(mgr.getClusterNodeLabelNames().isEmpty()); } @Test(timeout = 5000) public void testAddlabelWithCase() throws Exception { // Add some label, case will not ignore here - mgr.addToCluserNodeLabels(ImmutableSet.of("HeLlO")); - assertCollectionEquals(mgr.lastAddedlabels, Arrays.asList("HeLlO")); - Assert.assertFalse(mgr.getClusterNodeLabels().containsAll(Arrays.asList("hello"))); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("HeLlO")); + verifyNodeLabelAdded(Sets.newHashSet("HeLlO"), mgr.lastAddedlabels); + Assert.assertFalse(mgr.getClusterNodeLabelNames().containsAll( + Arrays.asList("hello"))); + } + + @Test(timeout = 5000) + public void testAddlabelWithExclusivity() throws Exception { + // Add some label, case will not ignore here + mgr.addToCluserNodeLabels(Arrays.asList(NodeLabel.newInstance("a", false), NodeLabel.newInstance("b", true))); + Assert.assertFalse(mgr.isExclusiveNodeLabel("a")); + Assert.assertTrue(mgr.isExclusiveNodeLabel("b")); } @Test(timeout = 5000) @@ -111,7 +119,7 @@ public void testAddInvalidlabel() throws IOException { try { Set set = new HashSet(); set.add(null); - mgr.addToCluserNodeLabels(set); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(set); } catch (IOException e) { caught = true; } @@ -119,7 +127,7 @@ public void testAddInvalidlabel() throws IOException { caught = false; try { - mgr.addToCluserNodeLabels(ImmutableSet.of(CommonNodeLabelsManager.NO_LABEL)); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of(CommonNodeLabelsManager.NO_LABEL)); } catch (IOException e) { caught = true; } @@ -128,7 +136,7 @@ public void testAddInvalidlabel() throws IOException { caught = false; try { - mgr.addToCluserNodeLabels(ImmutableSet.of("-?")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("-?")); } catch (IOException e) { caught = true; } @@ -136,7 +144,7 @@ public void testAddInvalidlabel() throws IOException { caught = false; try { - mgr.addToCluserNodeLabels(ImmutableSet.of(StringUtils.repeat("c", 257))); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of(StringUtils.repeat("c", 257))); } catch (IOException e) { caught = true; } @@ -144,7 +152,7 @@ public void testAddInvalidlabel() throws IOException { caught = false; try { - mgr.addToCluserNodeLabels(ImmutableSet.of("-aaabbb")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("-aaabbb")); } catch (IOException e) { caught = true; } @@ -152,7 +160,7 @@ public void testAddInvalidlabel() throws IOException { caught = false; try { - mgr.addToCluserNodeLabels(ImmutableSet.of("_aaabbb")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("_aaabbb")); } catch (IOException e) { caught = true; } @@ -160,7 +168,7 @@ public void testAddInvalidlabel() throws IOException { caught = false; try { - mgr.addToCluserNodeLabels(ImmutableSet.of("a^aabbb")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("a^aabbb")); } catch (IOException e) { caught = true; } @@ -168,7 +176,7 @@ public void testAddInvalidlabel() throws IOException { caught = false; try { - mgr.addToCluserNodeLabels(ImmutableSet.of("aa[a]bbb")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("aa[a]bbb")); } catch (IOException e) { caught = true; } @@ -198,7 +206,7 @@ public void testAddReplaceRemoveLabelsOnNodes() throws Exception { Assert.assertTrue("trying to add a empty node but succeeded", caught); // set node->label one by one - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p2"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n2"), toSet("p3"))); @@ -249,7 +257,7 @@ public void testAddReplaceRemoveLabelsOnNodes() throws Exception { @Test(timeout = 5000) public void testRemovelabelWithNodes() throws Exception { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n2"), toSet("p2"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n3"), toSet("p3"))); @@ -261,21 +269,21 @@ public void testRemovelabelWithNodes() throws Exception { mgr.removeFromClusterNodeLabels(ImmutableSet.of("p2", "p3")); Assert.assertTrue(mgr.getNodeLabels().isEmpty()); - Assert.assertTrue(mgr.getClusterNodeLabels().isEmpty()); + Assert.assertTrue(mgr.getClusterNodeLabelNames().isEmpty()); assertCollectionEquals(mgr.lastRemovedlabels, Arrays.asList("p2", "p3")); } @Test(timeout = 5000) public void testTrimLabelsWhenAddRemoveNodeLabels() throws IOException { - mgr.addToCluserNodeLabels(toSet(" p1")); - assertCollectionEquals(mgr.getClusterNodeLabels(), toSet("p1")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet(" p1")); + assertCollectionEquals(mgr.getClusterNodeLabelNames(), toSet("p1")); mgr.removeFromClusterNodeLabels(toSet("p1 ")); - Assert.assertTrue(mgr.getClusterNodeLabels().isEmpty()); + Assert.assertTrue(mgr.getClusterNodeLabelNames().isEmpty()); } @Test(timeout = 5000) public void testTrimLabelsWhenModifyLabelsOnNodes() throws IOException { - mgr.addToCluserNodeLabels(toSet(" p1", "p2")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet(" p1", "p2")); mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1 "))); assertMapEquals( mgr.getNodeLabels(), @@ -291,7 +299,7 @@ public void testTrimLabelsWhenModifyLabelsOnNodes() throws IOException { @Test(timeout = 5000) public void testReplaceLabelsOnHostsShouldUpdateNodesBelongTo() throws IOException { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"))); assertMapEquals( mgr.getNodeLabels(), @@ -330,7 +338,7 @@ public void testNodeLabelsDisabled() throws IOException { // add labels try { - mgr.addToCluserNodeLabels(ImmutableSet.of("x")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x")); } catch (IOException e) { assertNodeLabelsDisabledErrorMessage(e); caught = true; @@ -392,7 +400,7 @@ public void testNodeLabelsDisabled() throws IOException { @Test(timeout = 5000) public void testLabelsToNodes() throws IOException { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"))); Map> labelsToNodes = mgr.getLabelsToNodes(); assertLabelsToNodesEquals( @@ -453,7 +461,7 @@ public void testLabelsToNodes() @Test(timeout = 5000) public void testLabelsToNodesForSelectedLabels() throws IOException { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.addLabelsToNode( ImmutableMap.of( toNodeId("n1:1"), toSet("p1"), @@ -511,7 +519,7 @@ public void testNoMoreThanOneLabelExistedInOneHost() throws IOException { boolean failed = false; // As in YARN-2694, we temporarily disable no more than one label existed in // one host - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); try { mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1", "p2"))); } catch (IOException e) { @@ -538,30 +546,12 @@ public void testNoMoreThanOneLabelExistedInOneHost() throws IOException { Assert.assertTrue("Should failed when #labels > 1 on a host after add", failed); } - - @Test (timeout = 5000) - public void testUpdateNodeLabels() throws Exception { - boolean failed = false; - - // should fail: label isn't exist - try { - mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance( - "p1", false))); - } catch (YarnException e) { - failed = true; + + private void verifyNodeLabelAdded(Set expectedAddedLabelNames, + Collection addedNodeLabels) { + Assert.assertEquals(expectedAddedLabelNames.size(), addedNodeLabels.size()); + for (NodeLabel label : addedNodeLabels) { + Assert.assertTrue(expectedAddedLabelNames.contains(label.getNodeLabel())); } - Assert.assertTrue("Should fail since the node label doesn't exist", failed); - - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); - - mgr.updateNodeLabels(Arrays.asList( - NodeLabel.newInstance("p1", false), NodeLabel.newInstance("p2", true))); - Assert.assertEquals("p1", mgr.lastUpdatedNodeLabels.get(0).getNodeLabel()); - Assert.assertFalse(mgr.lastUpdatedNodeLabels.get(0).getIsExclusive()); - Assert.assertTrue(mgr.lastUpdatedNodeLabels.get(1).getIsExclusive()); - - // Check exclusive for p1/p2 - Assert.assertFalse(mgr.isExclusiveNodeLabel("p1")); - Assert.assertTrue(mgr.isExclusiveNodeLabel("p2")); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java index 6694290..7cf6af2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java @@ -84,9 +84,9 @@ public void after() throws IOException { @SuppressWarnings({ "unchecked", "rawtypes" }) @Test(timeout = 10000) public void testRecoverWithMirror() throws Exception { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); - mgr.addToCluserNodeLabels(toSet("p4")); - mgr.addToCluserNodeLabels(toSet("p5", "p6")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p4")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p5", "p6")); mgr.replaceLabelsOnNode((Map) ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"))); mgr.replaceLabelsOnNode((Map) ImmutableMap.of(toNodeId("n3"), toSet("p3"), @@ -110,8 +110,8 @@ public void testRecoverWithMirror() throws Exception { mgr.init(conf); // check variables - Assert.assertEquals(3, mgr.getClusterNodeLabels().size()); - Assert.assertTrue(mgr.getClusterNodeLabels().containsAll( + Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size()); + Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll( Arrays.asList("p2", "p4", "p6"))); assertMapContains(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n2"), @@ -129,8 +129,8 @@ public void testRecoverWithMirror() throws Exception { mgr.init(conf); // check variables - Assert.assertEquals(3, mgr.getClusterNodeLabels().size()); - Assert.assertTrue(mgr.getClusterNodeLabels().containsAll( + Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size()); + Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll( Arrays.asList("p2", "p4", "p6"))); assertMapContains(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n2"), @@ -147,9 +147,9 @@ public void testRecoverWithMirror() throws Exception { @SuppressWarnings({ "unchecked", "rawtypes" }) @Test(timeout = 10000) public void testEditlogRecover() throws Exception { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); - mgr.addToCluserNodeLabels(toSet("p4")); - mgr.addToCluserNodeLabels(toSet("p5", "p6")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p4")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p5", "p6")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"))); mgr.replaceLabelsOnNode((Map) ImmutableMap.of(toNodeId("n3"), toSet("p3"), @@ -173,8 +173,8 @@ public void testEditlogRecover() throws Exception { mgr.init(conf); // check variables - Assert.assertEquals(3, mgr.getClusterNodeLabels().size()); - Assert.assertTrue(mgr.getClusterNodeLabels().containsAll( + Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size()); + Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll( Arrays.asList("p2", "p4", "p6"))); assertMapContains(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n2"), @@ -191,9 +191,12 @@ public void testEditlogRecover() throws Exception { @SuppressWarnings({ "unchecked", "rawtypes" }) @Test (timeout = 10000) public void testSerilizationAfterRecovery() throws Exception { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); - mgr.addToCluserNodeLabels(toSet("p4")); - mgr.addToCluserNodeLabels(toSet("p5", "p6")); + // Add to cluster node labels, p2/p6 are non-exclusive. + mgr.addToCluserNodeLabels(Arrays.asList(NodeLabel.newInstance("p1", true), + NodeLabel.newInstance("p2", false), NodeLabel.newInstance("p3", true), + NodeLabel.newInstance("p4", true), NodeLabel.newInstance("p5", true), + NodeLabel.newInstance("p6", false))); + mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"))); mgr.replaceLabelsOnNode((Map) ImmutableMap.of(toNodeId("n3"), toSet("p3"), @@ -220,13 +223,6 @@ public void testSerilizationAfterRecovery() throws Exception { * p6: n6, n7 */ - mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance("p2", false))); - mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance("p6", false))); - - /* - * Set p2/p6 to be exclusive - */ - // shutdown mgr and start a new mgr mgr.stop(); @@ -235,8 +231,8 @@ public void testSerilizationAfterRecovery() throws Exception { mgr.start(); // check variables - Assert.assertEquals(3, mgr.getClusterNodeLabels().size()); - Assert.assertTrue(mgr.getClusterNodeLabels().containsAll( + Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size()); + Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll( Arrays.asList("p2", "p4", "p6"))); assertMapContains(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n2"), @@ -258,7 +254,7 @@ public void testSerilizationAfterRecovery() throws Exception { mgr = new MockNodeLabelManager(); mgr.init(conf); mgr.start(); - mgr.addToCluserNodeLabels(toSet("p7", "p8")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p7", "p8")); mgr.stop(); /* @@ -267,7 +263,7 @@ public void testSerilizationAfterRecovery() throws Exception { mgr = new MockNodeLabelManager(); mgr.init(conf); mgr.start(); - mgr.addToCluserNodeLabels(toSet("p9")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p9")); mgr.stop(); /* @@ -278,8 +274,8 @@ public void testSerilizationAfterRecovery() throws Exception { mgr.start(); // check variables - Assert.assertEquals(6, mgr.getClusterNodeLabels().size()); - Assert.assertTrue(mgr.getClusterNodeLabels().containsAll( + Assert.assertEquals(6, mgr.getClusterNodeLabelNames().size()); + Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll( Arrays.asList("p2", "p4", "p6", "p7", "p8", "p9"))); mgr.stop(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 12714de..773bee7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -77,8 +77,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; @@ -665,28 +663,6 @@ public ReplaceLabelsOnNodeResponse replaceLabelsOnNode( throw logAndWrapException(ioe, user.getShortUserName(), argName, msg); } } - - @Override - public UpdateNodeLabelsResponse updateNodeLabels( - UpdateNodeLabelsRequest request) throws YarnException, IOException { - String argName = "updateNodeLabels"; - final String msg = "update node labels"; - UserGroupInformation user = checkAcls(argName); - - checkRMStatus(user.getShortUserName(), argName, msg); - - UpdateNodeLabelsResponse response = UpdateNodeLabelsResponse.newInstance(); - - try { - rmContext.getNodeLabelManager().updateNodeLabels( - request.getNodeLabels()); - RMAuditLogger - .logSuccess(user.getShortUserName(), argName, "AdminService"); - return response; - } catch (YarnException ioe) { - throw logAndWrapException(ioe, user.getShortUserName(), argName, msg); - } - } private void checkRMStatus(String user, String argName, String msg) throws StandbyException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 21d70b4..cc9c6fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -1243,7 +1243,7 @@ public GetClusterNodeLabelsResponse getClusterNodeLabels( RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager(); GetClusterNodeLabelsResponse response = GetClusterNodeLabelsResponse.newInstance( - labelsMgr.getClusterNodeLabels()); + labelsMgr.getClusterNodeLabelNames()); return response; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index 1921195..b57fd12 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -136,7 +136,7 @@ public static void loadUpdateAndCheckCapacities(String queuePath, accessibleLabels.addAll(labels); } if (accessibleLabels.contains(CommonNodeLabelsManager.ANY)) { - accessibleLabels.addAll(mgr.getClusterNodeLabels()); + accessibleLabels.addAll(mgr.getClusterNodeLabelNames()); } accessibleLabels.add(CommonNodeLabelsManager.NO_LABEL); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 5ed6bb8..638c458 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -145,7 +145,7 @@ synchronized void setChildQueues(Collection childQueues) { " for children of queue " + queueName); } // check label capacities - for (String nodeLabel : labelManager.getClusterNodeLabels()) { + for (String nodeLabel : labelManager.getClusterNodeLabelNames()) { float capacityByLabel = queueCapacities.getCapacity(nodeLabel); // check children's labels float sum = 0; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 36f2b1d..7eee7a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -800,7 +800,7 @@ public NodeLabelsInfo getClusterNodeLabels(@Context HttpServletRequest hsr) NodeLabelsInfo ret = new NodeLabelsInfo(rm.getRMContext().getNodeLabelManager() - .getClusterNodeLabels()); + .getClusterNodeLabelNames()); return ret; } @@ -826,7 +826,7 @@ public Response addToClusterNodeLabels(final NodeLabelsInfo newNodeLabels, } rm.getRMContext().getNodeLabelManager() - .addToCluserNodeLabels(new HashSet( + .addToCluserNodeLabelsWithDefaultExclusivity(new HashSet( newNodeLabels.getNodeLabels())); return Response.status(Status.OK).build(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 430763c..0a49e52 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -200,7 +200,7 @@ protected ClientRMService createClientRMService() { }; rm.start(); RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager(); - labelsMgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + labelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); // Add a healthy node with label = x MockNM node = rm.registerNode("host1:1234", 1024); @@ -1407,7 +1407,7 @@ protected ClientRMService createClientRMService() { }; rm.start(); RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager(); - labelsMgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + labelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); Map> map = new HashMap>(); map.put(NodeId.newInstance("host1", 0), ImmutableSet.of("x")); @@ -1457,7 +1457,7 @@ protected ClientRMService createClientRMService() { }; rm.start(); RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager(); - labelsMgr.addToCluserNodeLabels(ImmutableSet.of("x", "y", "z")); + labelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "z")); Map> map = new HashMap>(); map.put(NodeId.newInstance("host1", 0), ImmutableSet.of("x")); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index a0b67f6..76dc522 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -2097,7 +2097,7 @@ protected RMNodeLabelsManager createNodeLabelManager() { clusterNodeLabels.add("y"); clusterNodeLabels.add("z"); // Add node label x,y,z - nodeLabelManager.addToCluserNodeLabels(clusterNodeLabels); + nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(clusterNodeLabels); // Add node Label to Node h1->x NodeId n1 = NodeId.newInstance("h1", 0); @@ -2122,7 +2122,7 @@ protected RMNodeLabelsManager createNodeLabelManager() { } Assert.assertEquals(clusterNodeLabels.size(), nodeLabelManager - .getClusterNodeLabels().size()); + .getClusterNodeLabelNames().size()); Map> nodeLabels = nodeLabelManager.getNodeLabels(); Assert.assertEquals(1, nodeLabelManager.getNodeLabels().size()); @@ -2141,7 +2141,7 @@ protected RMNodeLabelsManager createNodeLabelManager() { nodeLabelManager = rm2.getRMContext().getNodeLabelManager(); Assert.assertEquals(clusterNodeLabels.size(), nodeLabelManager - .getClusterNodeLabels().size()); + .getClusterNodeLabelNames().size()); nodeLabels = nodeLabelManager.getNodeLabels(); Assert.assertEquals(1, nodeLabelManager.getNodeLabels().size()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index 18d7df4..cc5f464 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -337,7 +337,7 @@ protected RMNodeLabelsManager createNodeLabelManager() { rm.start(); try { - nodeLabelsMgr.addToCluserNodeLabels(toSet("A", "B", "C")); + nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("A", "B", "C")); } catch (IOException e) { Assert.fail("Caught Exception while intializing"); e.printStackTrace(); @@ -386,7 +386,7 @@ protected RMNodeLabelsManager createNodeLabelManager() { rm.start(); try { - nodeLabelsMgr.addToCluserNodeLabels(toSet("X", "Y", "Z")); + nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("X", "Y", "Z")); } catch (IOException e) { Assert.fail("Caught Exception while intializing"); e.printStackTrace(); @@ -439,7 +439,7 @@ protected RMNodeLabelsManager createNodeLabelManager() { rm.start(); try { - nodeLabelsMgr.addToCluserNodeLabels(toSet("X", "Y", "Z")); + nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("X", "Y", "Z")); } catch (IOException e) { Assert.fail("Caught Exception while intializing"); e.printStackTrace(); @@ -491,7 +491,7 @@ protected RMNodeLabelsManager createNodeLabelManager() { }; rm.start(); try { - nodeLabelsMgr.addToCluserNodeLabels(toSet("A", "B", "C")); + nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("A", "B", "C")); } catch (IOException e) { Assert.fail("Caught Exception while intializing"); e.printStackTrace(); @@ -551,7 +551,7 @@ protected RMNodeLabelsManager createNodeLabelManager() { rm.start(); // adding valid labels try { - nodeLabelsMgr.addToCluserNodeLabels(toSet("A", "B", "C")); + nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("A", "B", "C")); } catch (IOException e) { Assert.fail("Caught Exception while intializing"); e.printStackTrace(); @@ -636,7 +636,7 @@ protected RMNodeLabelsManager createNodeLabelManager() { rm.start(); try { - nodeLabelsMgr.addToCluserNodeLabels(toSet("A", "B", "C")); + nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("A", "B", "C")); } catch (IOException e) { Assert.fail("Caught Exception while intializing"); e.printStackTrace(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java index 14d96a0..9548029 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java @@ -57,7 +57,8 @@ public void updateNodeToLabelsMappings( } @Override - public void storeNewClusterNodeLabels(Set label) throws IOException { + public void storeNewClusterNodeLabels(List label) + throws IOException { // do nothing } @@ -65,12 +66,6 @@ public void storeNewClusterNodeLabels(Set label) throws IOException { public void close() throws IOException { // do nothing } - - @Override - public void updateNodeLabels(List updatedNodeLabels) - throws IOException { - // do nothing - } }; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java index ace8742..05bb1e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java @@ -63,7 +63,7 @@ public void after() { @Test(timeout = 5000) public void testGetLabelResourceWhenNodeActiveDeactive() throws Exception { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"), toNodeId("n3"), toSet("p3"))); @@ -81,7 +81,7 @@ public void testGetLabelResourceWhenNodeActiveDeactive() throws Exception { // check add labels multiple times shouldn't overwrite // original attributes on labels like resource - mgr.addToCluserNodeLabels(toSet("p1", "p4")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p4")); Assert.assertEquals(mgr.getResourceByLabel("p1", null), Resources.add(SMALL_RESOURCE, LARGE_NODE)); Assert.assertEquals(mgr.getResourceByLabel("p4", null), EMPTY_RESOURCE); @@ -120,7 +120,7 @@ public void testActivateNodeManagerWithZeroPort() throws Exception { @SuppressWarnings({ "unchecked", "rawtypes" }) @Test(timeout = 5000) public void testGetLabelResource() throws Exception { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"), toNodeId("n3"), toSet("p3"))); @@ -137,7 +137,7 @@ public void testGetLabelResource() throws Exception { Assert.assertEquals(mgr.getResourceByLabel("p3", null), SMALL_RESOURCE); // add more labels - mgr.addToCluserNodeLabels(toSet("p4", "p5", "p6")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p4", "p5", "p6")); mgr.replaceLabelsOnNode((Map) ImmutableMap.of(toNodeId("n4"), toSet("p1"), toNodeId("n5"), toSet("p2"), toNodeId("n6"), toSet("p3"), toNodeId("n7"), toSet("p4"), toNodeId("n8"), toSet("p5"))); @@ -216,7 +216,7 @@ public void testGetQueueResource() throws Exception { * host3 : yellow * host4 : */ - mgr.addToCluserNodeLabels(toSet("red", "blue", "yellow")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("red", "blue", "yellow")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host1"), toSet("red"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host2"), toSet("blue"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host3"), toSet("yellow"))); @@ -397,7 +397,7 @@ public void testGetLabelResourceWhenMultipleNMsExistingInSameHost() throws IOExc Resources.multiply(SMALL_RESOURCE, 4)); // change two of these nodes to p1, check resource of no_label and P1 - mgr.addToCluserNodeLabels(toSet("p1")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1")); mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"), toNodeId("n1:2"), toSet("p1"))); @@ -412,7 +412,7 @@ public void testGetLabelResourceWhenMultipleNMsExistingInSameHost() throws IOExc @Test(timeout = 5000) public void testRemoveLabelsFromNode() throws Exception { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"), toNodeId("n3"), toSet("p3"))); // active one NM to n1:1 @@ -432,7 +432,7 @@ public void testRemoveLabelsFromNode() throws Exception { @Test(timeout = 5000) public void testGetLabelsOnNodesWhenNodeActiveDeactive() throws Exception { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of( toNodeId("n1"), toSet("p2"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"))); @@ -489,7 +489,7 @@ private void checkNodeLabelInfo(List infos, String labelName, int a @Test(timeout = 5000) public void testPullRMNodeLabelsInfo() throws IOException { - mgr.addToCluserNodeLabels(toSet("x", "y", "z")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("x", "y", "z")); mgr.activateNode(NodeId.newInstance("n1", 1), Resource.newInstance(10, 0)); mgr.activateNode(NodeId.newInstance("n2", 1), Resource.newInstance(10, 0)); mgr.activateNode(NodeId.newInstance("n3", 1), Resource.newInstance(10, 0)); @@ -516,7 +516,7 @@ public void testLabelsToNodesOnNodeActiveDeactive() throws Exception { mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels())); // Add labels and replace labels on node - mgr.addToCluserNodeLabels(toSet("p1")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"))); // p1 -> n1, n1:1 Assert.assertEquals(2, mgr.getLabelsToNodes().get("p1").size()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index aaa615d..8d7874f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -2578,7 +2578,7 @@ public void testQueueHierarchyPendingResourceUpdate() throws Exception { final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.init(conf); - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"))); MemoryRMStateStore memStore = new MemoryRMStateStore(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java index 1e339d9..c5439d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java @@ -98,7 +98,7 @@ private void checkUsedResource(MockRM rm, String queueName, int memory, @Test (timeout = 30000) public void testNodeUpdate() throws Exception { // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y", "z")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "z")); // set mapping: // h1 -> x diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index 03b8f5c..46ab6a1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -399,7 +399,7 @@ public void testContainerAllocationWithSingleUserLimits() throws Exception { mgr.init(conf); // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"), NodeId.newInstance("h2", 0), toSet("y"))); @@ -482,7 +482,7 @@ public void testContainerAllocateWithComplexLabels() throws Exception { */ // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y", "z")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "z")); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"), NodeId.newInstance("h2", 0), toSet("y"), NodeId.newInstance("h3", 0), toSet("y"), NodeId.newInstance("h4", 0), @@ -566,7 +566,7 @@ public RMNodeLabelsManager createNodeLabelManager() { @Test (timeout = 120000) public void testContainerAllocateWithLabels() throws Exception { // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"), NodeId.newInstance("h2", 0), toSet("y"))); @@ -642,7 +642,7 @@ public void testContainerAllocateWithDefaultQueueLabels() throws Exception { // instead, it uses default queue label expression // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"), NodeId.newInstance("h2", 0), toSet("y"))); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index 34f7c2c..0206772 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -417,7 +417,7 @@ private void setupQueueConfigurationWithSingleLevel( @Test public void testQueueParsingReinitializeWithLabels() throws IOException { - nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue")); + nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("red", "blue")); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); setupQueueConfigurationWithoutLabels(csConf); @@ -505,7 +505,7 @@ private void checkQueueLabels(CapacityScheduler capacityScheduler) { @Test public void testQueueParsingWithLabels() throws IOException { - nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue")); + nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("red", "blue")); YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = @@ -529,7 +529,7 @@ public void testQueueParsingWithLabels() throws IOException { @Test public void testQueueParsingWithLabelsInherit() throws IOException { - nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue")); + nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("red", "blue")); YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = @@ -668,7 +668,7 @@ public void testQueueParsingWithUnusedLabels() throws IOException { // Initialize a cluster with labels, but doesn't use them, reinitialize // shouldn't fail - nodeLabelManager.addToCluserNodeLabels(labels); + nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(labels); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();