From 21a88bae2536b702e902d943a8d926b5be68e372 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Sun, 2 Sep 2018 12:13:49 +0530 Subject: [PATCH] YARN-8739 --- .../src/main/java/org/apache/hadoop/ha/HAAdmin.java | 5 ++++- .../hadoop/yarn/sls/scheduler/RMNodeWrapper.java | 2 +- .../protocolrecords/GetAttributesToNodesResponse.java | 5 +++-- .../api/ResourceManagerAdministrationProtocol.java | 2 +- .../NodesToAttributesMappingResponse.java | 4 ++++ .../org/apache/hadoop/yarn/client/api/YarnClient.java | 3 ++- .../impl/pb/GetAttributesToNodesResponsePBImpl.java | 8 +++++--- .../yarn/nodelabels/FileSystemNodeLabelsStore.java | 4 +++- .../hadoop/yarn/nodelabels/NodeAttributesManager.java | 3 ++- .../yarn/nodelabels/NonAppendableFSNodeLabelStore.java | 2 +- .../org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java | 3 ++- .../yarn/nodelabels/store/AbstractFSNodeStore.java | 8 ++++---- .../hadoop/yarn/nodelabels/store/FSStoreOpHandler.java | 18 +++++++++++++----- .../apache/hadoop/yarn/nodelabels/store/StoreOp.java | 4 ++-- .../yarn/nodelabels/store/op/AddClusterLabelOp.java | 4 ++-- .../yarn/nodelabels/store/op/NodeLabelMirrorOp.java | 3 +++ .../hadoop/yarn/nodelabels/store/op/NodeToLabelOp.java | 4 ++-- .../yarn/nodelabels/store/op/RemoveClusterLabelOp.java | 4 ++-- .../impl/pb/NodeToAttributesPBImpl.java | 3 +++ .../impl/pb/NodesToAttributesMappingRequestPBImpl.java | 3 +++ .../pb/NodesToAttributesMappingResponsePBImpl.java | 3 +++ .../pb/RemoveFromClusterNodeLabelsRequestPBImpl.java | 3 +++ .../org/apache/hadoop/yarn/api/TestPBImplRecords.java | 2 +- .../yarn/nodelabels/TestFileSystemNodeLabelsStore.java | 2 +- .../hadoop/yarn/nodelabels/TestNodeLabelUtil.java | 8 ++++++-- .../hadoop/yarn/server/MockResourceManagerFacade.java | 11 ++++++----- .../nodemanager/nodelabels/NodeAttributesProvider.java | 2 -- .../nodemanager/TestNodeStatusUpdaterForLabels.java | 3 +++ .../server/resourcemanager/ResourceTrackerService.java | 1 - .../nodelabels/NodeAttributesManagerImpl.java | 7 ++++--- .../nodelabels/NodeAttributesStoreEvent.java | 3 ++- .../hadoop/yarn/server/resourcemanager/MockNodes.java | 2 +- .../nodelabels/TestFileSystemNodeAttributeStore.java | 3 +++ 33 files changed, 95 insertions(+), 47 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java index 8c92bd04471..e955979acde 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java @@ -574,7 +574,10 @@ private int help(String[] argv) { } return 0; } - + + /** + * UsageInfo class holds args and help details. + */ public static class UsageInfo { public final String args; public final String help; diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java index 248b634b170..c73fb15be00 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java @@ -207,7 +207,7 @@ public Integer getDecommissioningTimeout() { public Map getAllocationTagsWithCount() { return node.getAllocationTagsWithCount(); } - + @Override public Set getAllNodeAttributes() { return node.getAllNodeAttributes(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java index 9bd529f3c4c..c83785fc6ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java @@ -54,12 +54,13 @@ public abstract void setAttributeToNodes( /** * Get mapping of NodeAttributeKey to its associated mapping of list of - * NodeToAttributeValuenode to attribute value. + * NodeToAttributeValue associated with attribute. * * @return Map> node attributes * to list of NodeToAttributeValuenode. */ @Public @Evolving - public abstract Map> getAttributesToNodes(); + public abstract Map> getAttributesToNodes(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java index 58bb270f9cf..4777cf8b62a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java @@ -150,7 +150,7 @@ public RefreshClusterMaxPriorityResponse refreshClusterMaxPriority( @Private @Idempotent - public NodesToAttributesMappingResponse mapAttributesToNodes( + NodesToAttributesMappingResponse mapAttributesToNodes( NodesToAttributesMappingRequest request) throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingResponse.java index 8e44adfc6a3..10081e1f928 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingResponse.java @@ -20,6 +20,10 @@ import org.apache.hadoop.yarn.util.Records; +/** + * NodesToAttributesMappingResponse holds response object for attribute + * mapping. + */ public class NodesToAttributesMappingResponse { public static NodesToAttributesMappingResponse newInstance() { return Records.newRecord(NodesToAttributesMappingResponse.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index f51b2f9b168..59fa6a8f2f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -936,7 +936,8 @@ public abstract Resource getResourceProfile(String profile) */ @Public @Unstable - public abstract Map> getAttributesToNodes( + public abstract Map> getAttributesToNodes( Set attributes) throws YarnException, IOException; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesResponsePBImpl.java index 11999bdc1e8..0d4c6e0d812 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesResponsePBImpl.java @@ -100,8 +100,9 @@ private void addAttributesToNodesToProto() { Iterable iterable = () -> new Iterator() { - private Iterator>> iter = - attributesToNodes.entrySet().iterator(); + private Iterator>> iter = attributesToNodes.entrySet() + .iterator(); @Override public void remove() { @@ -198,7 +199,8 @@ public void setAttributeToNodes( } @Override - public Map> getAttributesToNodes() { + public Map> getAttributesToNodes() { initAttributesToNodes(); return this.attributesToNodes; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java index e11e6f82377..6c459c2739d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java @@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.nodelabels.store.AbstractFSNodeStore; -import org.apache.hadoop.yarn.nodelabels.store.FSStoreOpHandler; import org.apache.hadoop.yarn.nodelabels.store.op.AddClusterLabelOp; import org.apache.hadoop.yarn.nodelabels.store.FSStoreOpHandler.StoreType; @@ -41,6 +40,9 @@ import java.util.Map; import java.util.Set; +/** + * FileSystemNodeLabelsStore for storing node labels. + */ public class FileSystemNodeLabelsStore extends AbstractFSNodeStore implements NodeLabelsStore { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java index 20f72d9422f..ca04e8da96e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java @@ -96,7 +96,8 @@ public abstract void removeNodeAttributes( * @return a Map of attributeKeys to a map of hostnames to its attribute * values. */ - public abstract Map> getAttributesToNodes( + public abstract Map> getAttributesToNodes( Set attributes); /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NonAppendableFSNodeLabelStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NonAppendableFSNodeLabelStore.java index 29bfff933d9..674703763ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NonAppendableFSNodeLabelStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NonAppendableFSNodeLabelStore.java @@ -36,7 +36,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; /** - * Store implementation for Non Appendable File Store + * Store implementation for Non Appendable File Store. */ public class NonAppendableFSNodeLabelStore extends FileSystemNodeLabelsStore { protected static final Log diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java index a8d151c0398..5e755f4daeb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java @@ -30,7 +30,8 @@ /** * Partition representation in RM. */ -public class RMNodeLabel extends AbstractLabel implements Comparable { +public class RMNodeLabel extends AbstractLabel + implements Comparable { private boolean exclusive; private NodeLabel nodeLabel; private Set nodeIds; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/AbstractFSNodeStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/AbstractFSNodeStore.java index 216fc79ecbd..7127d112df3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/AbstractFSNodeStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/AbstractFSNodeStore.java @@ -182,14 +182,14 @@ protected void recoverFromStore() throws IOException { LOG.info("Finished create editlog file at:" + editLogPath.toString()); } - protected void loadManagerFromEditLog(Path editLogPath) throws IOException { - if (!fs.exists(editLogPath)) { + protected void loadManagerFromEditLog(Path editPath) throws IOException { + if (!fs.exists(editPath)) { return; } - try (FSDataInputStream is = fs.open(editLogPath)) { + try (FSDataInputStream is = fs.open(editPath)) { while (true) { try { - StoreOp storeOp = FSStoreOpHandler.get(is.readInt(),storeType); + StoreOp storeOp = FSStoreOpHandler.get(is.readInt(), storeType); storeOp.recover(is, manager); } catch (EOFException e) { // EOF hit, break diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/FSStoreOpHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/FSStoreOpHandler.java index a626537aa81..59a1860e315 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/FSStoreOpHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/FSStoreOpHandler.java @@ -41,6 +41,9 @@ editLogOp; private static Map> mirrorOp; + /** + * Store Type enum to hold label and attribute. + */ public enum StoreType { NODE_LABEL_STORE, NODE_ATTRIBUTE @@ -53,14 +56,19 @@ // registerLog edit log operation //Node Label Operations - registerLog(NODE_LABEL_STORE, AddClusterLabelOp.OPCODE, AddClusterLabelOp.class); + registerLog(NODE_LABEL_STORE, AddClusterLabelOp.OPCODE, + AddClusterLabelOp.class); registerLog(NODE_LABEL_STORE, NodeToLabelOp.OPCODE, NodeToLabelOp.class); - registerLog(NODE_LABEL_STORE, RemoveClusterLabelOp.OPCODE, RemoveClusterLabelOp.class); + registerLog(NODE_LABEL_STORE, RemoveClusterLabelOp.OPCODE, + RemoveClusterLabelOp.class); //NodeAttibute operation - registerLog(NODE_ATTRIBUTE, AddNodeToAttributeLogOp.OPCODE, AddNodeToAttributeLogOp.class); - registerLog(NODE_ATTRIBUTE, RemoveNodeToAttributeLogOp.OPCODE, RemoveNodeToAttributeLogOp.class); - registerLog(NODE_ATTRIBUTE, ReplaceNodeToAttributeLogOp.OPCODE, ReplaceNodeToAttributeLogOp.class); + registerLog(NODE_ATTRIBUTE, AddNodeToAttributeLogOp.OPCODE, + AddNodeToAttributeLogOp.class); + registerLog(NODE_ATTRIBUTE, RemoveNodeToAttributeLogOp.OPCODE, + RemoveNodeToAttributeLogOp.class); + registerLog(NODE_ATTRIBUTE, ReplaceNodeToAttributeLogOp.OPCODE, + ReplaceNodeToAttributeLogOp.class); // registerLog Mirror op diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/StoreOp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/StoreOp.java index c26e1dc40ee..e0b26da82e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/StoreOp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/StoreOp.java @@ -30,7 +30,7 @@ public interface StoreOp { /** - * Write operation to persistent storage + * Write operation to persistent storage. * * @param write write to be done to * @param mgr manager used by store @@ -39,7 +39,7 @@ void write(W write, M mgr) throws IOException; /** - * Read and populate StoreOp + * Read and populate StoreOp. * * @param read read to be done from * @param mgr manager used by store diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/AddClusterLabelOp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/AddClusterLabelOp.java index ce736aae085..f0259b37e62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/AddClusterLabelOp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/AddClusterLabelOp.java @@ -57,8 +57,8 @@ public void recover(InputStream is, CommonNodeLabelsManager mgr) mgr.addToCluserNodeLabels(labels); } - public AddClusterLabelOp setLabels(List labels) { - this.labels = labels; + public AddClusterLabelOp setLabels(List nodeLabels) { + this.labels = nodeLabels; return this; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeLabelMirrorOp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeLabelMirrorOp.java index 3ec837b0b3d..95004866458 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeLabelMirrorOp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeLabelMirrorOp.java @@ -35,6 +35,9 @@ import java.util.Map; import java.util.Set; +/** + * NodeLabel Mirror Op class. + */ public class NodeLabelMirrorOp extends FSNodeStoreLogOp { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeToLabelOp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeToLabelOp.java index 0e1e3985114..df08ffcc15c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeToLabelOp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeToLabelOp.java @@ -59,8 +59,8 @@ public void recover(InputStream is, CommonNodeLabelsManager mgr) } public NodeToLabelOp setNodeToLabels( - Map> nodeToLabels) { - this.nodeToLabels = nodeToLabels; + Map> nodeToLabelsList) { + this.nodeToLabels = nodeToLabelsList; return this; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/RemoveClusterLabelOp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/RemoveClusterLabelOp.java index 4f6d4bdac88..2fc4ac3dfc2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/RemoveClusterLabelOp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/RemoveClusterLabelOp.java @@ -59,8 +59,8 @@ public void recover(InputStream is, CommonNodeLabelsManager mgr) mgr.removeFromClusterNodeLabels(labels); } - public RemoveClusterLabelOp setLabels(Collection labels) { - this.labels = labels; + public RemoveClusterLabelOp setLabels(Collection nodeLabels) { + this.labels = nodeLabels; return this; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeToAttributesPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeToAttributesPBImpl.java index 7204914ace5..94b967b4adf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeToAttributesPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeToAttributesPBImpl.java @@ -28,6 +28,9 @@ import org.apache.hadoop.yarn.proto.YarnProtos.NodeToAttributesProtoOrBuilder; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes; +/** + * Proto class for Node to attributes mapping. + */ public class NodeToAttributesPBImpl extends NodeToAttributes { private NodeToAttributesProto proto = NodeToAttributesProto.getDefaultInstance(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodesToAttributesMappingRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodesToAttributesMappingRequestPBImpl.java index 6cb9a97632e..c1a86b3cd3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodesToAttributesMappingRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodesToAttributesMappingRequestPBImpl.java @@ -29,6 +29,9 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes; import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest; +/** + * Proto class for node to attributes mapping request. + */ public class NodesToAttributesMappingRequestPBImpl extends NodesToAttributesMappingRequest { private NodesToAttributesMappingRequestProto proto = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodesToAttributesMappingResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodesToAttributesMappingResponsePBImpl.java index 0554ef849d9..955c3cbadfe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodesToAttributesMappingResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodesToAttributesMappingResponsePBImpl.java @@ -21,6 +21,9 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodesToAttributesMappingResponseProto; import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingResponse; +/** + * Proto class for node to attributes mapping response. + */ public class NodesToAttributesMappingResponsePBImpl extends NodesToAttributesMappingResponse { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RemoveFromClusterNodeLabelsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RemoveFromClusterNodeLabelsRequestPBImpl.java index f6338045f47..afabcd919fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RemoveFromClusterNodeLabelsRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RemoveFromClusterNodeLabelsRequestPBImpl.java @@ -28,6 +28,9 @@ import org.apache.hadoop.yarn.server.api.protocolrecords .RemoveFromClusterNodeLabelsRequest; +/** + * Proto class to handlde RemoveFromClusterNodeLabels request. + */ public class RemoveFromClusterNodeLabelsRequestPBImpl extends RemoveFromClusterNodeLabelsRequest { Set labels; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 682d6ec68ef..cdb6c4664fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -465,7 +465,7 @@ public static void setup() throws Exception { generateByNewInstance(NodeToAttributeValue.class); generateByNewInstance(NodeAttributeInfo.class); generateByNewInstance(NodesToAttributesMappingRequest.class); - } + } @Test public void testAllocateRequestPBImpl() throws Exception { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java index 93c039a5bc3..36dbc2b9bc8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java @@ -360,7 +360,7 @@ private void verifyMkdirsCount(FileSystemNodeLabelsStore store, Mockito.when(store.getFs().exists(Mockito.any( Path.class))).thenReturn(existsRetVal); store.init(conf, mgr); - Mockito.verify(store.getFs(),Mockito.times( + Mockito.verify(store.getFs(), Mockito.times( expectedNumOfCalls)).mkdirs(Mockito.any(Path .class)); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestNodeLabelUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestNodeLabelUtil.java index d43da4fb95d..afdfcbbb5f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestNodeLabelUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestNodeLabelUtil.java @@ -20,12 +20,16 @@ import static org.junit.Assert.fail; import org.junit.Test; +/** + * Test class to verify node label util ops. + */ public class TestNodeLabelUtil { @Test public void testAttributeValueAddition() { String[] values = - new String[] {"1_8", "1.8", "ABZ", "ABZ", "az", "a-z","a_z", "123456789"}; + new String[]{"1_8", "1.8", "ABZ", "ABZ", "az", "a-z", "a_z", + "123456789"}; for (String val : values) { try { NodeLabelUtil.checkAndThrowAttributeValue(val); @@ -34,7 +38,7 @@ public void testAttributeValueAddition() { } } - String[] invalidVals = new String[] {"_18","1,8","1/5",".15","1\\5"}; + String[] invalidVals = new String[]{"_18", "1,8", "1/5", ".15", "1\\5"}; for (String val : invalidVals) { try { NodeLabelUtil.checkAndThrowAttributeValue(val); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java index 88f442a2bdd..ab8b6ff8dd7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java @@ -188,8 +188,8 @@ private HashSet applicationMap = new HashSet<>(); private HashSet keepContainerOnUams = new HashSet<>(); - private HashMap> applicationContainerIdMap = - new HashMap>(); + private HashMap> applicationContainerIdMap = new HashMap<>(); private AtomicInteger containerIndex = new AtomicInteger(0); private Configuration conf; private int subClusterId; @@ -494,7 +494,7 @@ public SubmitApplicationResponse submitApplication( if (request.getApplicationSubmissionContext().getUnmanagedAM() || request.getApplicationSubmissionContext() .getKeepContainersAcrossApplicationAttempts()) { - keepContainerOnUams.add(appId); + keepContainerOnUams.add(appId); } return SubmitApplicationResponse.newInstance(); } @@ -920,8 +920,9 @@ public GetNodesToAttributesResponse getNodesToAttributes( } @Override - public NodesToAttributesMappingResponse mapAttributesToNodes(NodesToAttributesMappingRequest request) - throws YarnException, IOException { + public NodesToAttributesMappingResponse mapAttributesToNodes( + NodesToAttributesMappingRequest request) + throws YarnException, IOException { return null; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeAttributesProvider.java index 824002451eb..2a418009406 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeAttributesProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeAttributesProvider.java @@ -19,8 +19,6 @@ import org.apache.hadoop.yarn.api.records.NodeAttribute; -import java.util.Set; - /** * Abstract class which will be responsible for fetching the node attributes. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java index 3e2d963e898..a86ca3e8211 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java @@ -180,6 +180,9 @@ public UnRegisterNodeManagerResponse unRegisterNodeManager( } } + /** + * A dummy NodeLabelsProvider class for tests. + */ public static class DummyNodeLabelsProvider extends NodeLabelsProvider { public DummyNodeLabelsProvider() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 9c4d594d8a7..b67172e01cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -515,7 +515,6 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) * 4. Send healthStatus to RMNode * 5. Update node's labels if distributed Node Labels configuration is enabled */ - NodeId nodeId = remoteNodeStatus.getNodeId(); // 1. Check if it's a valid (i.e. not excluded) node, if not, see if it is diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java index 9cef87e52e5..43491d4dcee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java @@ -162,8 +162,8 @@ private void internalUpdateAttributesOnNodes( StringBuilder logMsg = new StringBuilder(op.name()); logMsg.append(" attributes on nodes:"); // do update labels from nodes - for (Entry> entry : nodeAttributeMapping - .entrySet()) { + for (Entry> entry : + nodeAttributeMapping.entrySet()) { String nodeHost = entry.getKey(); Map attributes = entry.getValue(); @@ -399,7 +399,8 @@ protected String normalizeAttributeValue(String value) { } @Override - public Map> getAttributesToNodes( + public Map> getAttributesToNodes( Set attributes) { try { readLock.lock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesStoreEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesStoreEvent.java index d04e8cf224e..707981f75c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesStoreEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesStoreEvent.java @@ -41,7 +41,8 @@ public NodeAttributesStoreEvent( this.operation = operation; } - public Map> getNodeAttributeMappingList() { + public Map> getNodeAttributeMappingList() { return nodeAttributeMapping; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index 7c822b49f0c..a8719932f3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -285,7 +285,7 @@ public Integer getDecommissioningTimeout() { public Map getAllocationTagsWithCount() { return null; } - + public void setNodeAttributes(String prefix, Set nodeAttributes) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java index 502f9d051c7..07ea20ddda7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java @@ -36,6 +36,9 @@ import java.util.Map; import java.util.Set; +/** + * Test class for FileSystemNodeAttributeStore. + */ public class TestFileSystemNodeAttributeStore { private MockNodeAttrbuteManager mgr = null; -- 2.15.2 (Apple Git-101.1)