diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java index 01c70b296eb..90b880e91da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java @@ -46,6 +46,7 @@ public abstract class NodeAttribute { public static final String DEFAULT_PREFIX = ""; + public static final String PREFIX_DISTRIBUTED = "yarn.dist.io"; public static NodeAttribute newInstance(String attributeName, NodeAttributeType attributeType, String attributeValue) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/InvalidNodeAttributeException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/InvalidNodeAttributeException.java new file mode 100644 index 00000000000..f794d9272bd --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/InvalidNodeAttributeException.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.exceptions; + +/** + * Class to encapsulate node attribute validation exceptions. + */ +public class InvalidNodeAttributeException extends YarnException { + + public InvalidNodeAttributeException(String message) { + super(message); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/NodeAttributeUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/NodeAttributeUtils.java new file mode 100644 index 00000000000..ec9665ee749 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/NodeAttributeUtils.java @@ -0,0 +1,128 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.util; + +import com.google.common.base.Strings; +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.exceptions.InvalidNodeAttributeException; + +import java.util.HashSet; +import java.util.Set; + +/** + * Utility class for node attributes helper functions. + */ +@InterfaceAudience.Public +@InterfaceStability.Evolving +public final class NodeAttributeUtils { + + /** + * A node attribute identifier differentiates an attribute with another + * by its name and prefix. + */ + private final static class NodeAttributeIdentifier { + + private final String attributePrefix; + private final String attributeName; + + private NodeAttributeIdentifier(String prefix, String name) { + this.attributePrefix = prefix; + this.attributeName = name; + } + + protected static NodeAttributeIdentifier newInstance( + String prefix, String name) { + return new NodeAttributeIdentifier(prefix, name); + } + + protected String getAttributePrefix() { + return attributePrefix; + } + + protected String getAttributeName() { + return attributeName; + } + + @Override + public int hashCode() { + return new HashCodeBuilder(15, 17) + .append(getAttributeName()) + .append(getAttributePrefix()) + .toHashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj != null && + obj instanceof NodeAttributeIdentifier) { + NodeAttributeIdentifier target = (NodeAttributeIdentifier) obj; + if (getAttributePrefix() == null + && target.getAttributePrefix() == null) { + if (getAttributeName().equals(target.getAttributeName())) { + return true; + } + } + if (getAttributeName().equals(target.getAttributeName()) && + getAttributePrefix().equals(target.getAttributePrefix())) { + return true; + } + } + return false; + } + } + + /** + * Validate if a given set of attributes are valid. Attributes could be + * invalid if any of following conditions is met: + * + * + * @param attributeSet + * @throws InvalidNodeAttributeException + */ + public static void validateNodeAttributes(Set attributeSet) + throws InvalidNodeAttributeException { + Set attributeIdentifiers = new HashSet<>(); + if (attributeSet != null) { + + for (NodeAttribute attribute : attributeSet) { + String prefix = attribute.getAttributePrefix(); + if (Strings.isNullOrEmpty(prefix)) { + throw new InvalidNodeAttributeException( + "Attribute prefix must be set"); + } + + NodeAttributeIdentifier identifier = NodeAttributeIdentifier + .newInstance(prefix, attribute.getAttributeName()); + if (!attributeIdentifiers.add(identifier)) { + throw new InvalidNodeAttributeException( + "Duplicated attributes found. Attribute that has same" + + " prefix/name already exists. Duplicate entry: " + + attribute.toString()); + } + } + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java index 06771bae28c..169881d49fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java @@ -20,6 +20,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.NodeAttribute; import org.apache.hadoop.yarn.api.records.NodeAttributeType; +import org.apache.hadoop.yarn.exceptions.InvalidNodeAttributeException; +import org.apache.hadoop.yarn.util.NodeAttributeUtils; import java.io.IOException; import java.util.HashSet; @@ -116,13 +118,25 @@ public TimerTask createTimerTask() { + NODE_ATTRIBUTE_DELIMITER + "ATTRIBUTE_VALUE; but get " + nodeAttribute); } - NodeAttribute na = NodeAttribute - .newInstance(attributeStrs[0], - NodeAttributeType.valueOf(attributeStrs[1]), - attributeStrs[2]); + // Automatically setup prefix for collected attributes + NodeAttribute na = NodeAttribute.newInstance( + NodeAttribute.PREFIX_DISTRIBUTED, + attributeStrs[0], + NodeAttributeType.valueOf(attributeStrs[1]), + attributeStrs[2]); attributeSet.add(na); } } + + // Before updating the attributes to the provider, + // verify if they are valid + try { + NodeAttributeUtils.validateNodeAttributes(attributeSet); + } catch (InvalidNodeAttributeException e) { + throw new IOException("Node attributes collected by the script " + + "contains some invalidate entries. Detail message: " + + e.getMessage()); + } return attributeSet; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java index 58d2d20916b..f764626b4eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java @@ -220,4 +220,31 @@ public void testFetchInterval() throws Exception { } }, 500, 3000); } + + @Test + public void testNodeAttributesValidation() throws Exception{ + // Script output contains ambiguous node attributes + String scriptContent = "echo NODE_ATTRIBUTE:host,STRING,host1234\n " + + "echo NODE_ATTRIBUTE:host,STRING,host2345\n " + + "echo NODE_ATTRIBUTE:ip,STRING,10.0.0.1"; + + writeNodeAttributeScriptFile(scriptContent, true); + + nodeAttributesProvider.init(getConfForNodeAttributeScript()); + nodeAttributesProvider.start(); + + // There should be no attributes found, and we should + // see Malformed output warnings in the log + try { + GenericTestUtils + .waitFor(() -> nodeAttributesProvider + .getDescriptors().size() == 3, + 500, 3000); + Assert.fail("This test should timeout because the provide is unable" + + " to parse any attributes from the script output."); + } catch (TimeoutException e) { + Assert.assertEquals(0, nodeAttributesProvider + .getDescriptors().size()); + } + } }