diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java index 279f127..9ef2e06 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java @@ -18,6 +18,9 @@ package org.apache.hadoop.yarn.api.records; +import java.nio.ByteBuffer; +import java.util.Map; + import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; @@ -79,6 +82,21 @@ public static Container newInstance(ContainerId containerId, NodeId nodeId, return container; } + @Private + @Unstable + public static Container newInstance(ContainerId containerId, NodeId nodeId, + String nodeHttpAddress, Resource resource, Priority priority, + Token containerToken, Map servicesMetaData) { + Container container = Records.newRecord(Container.class); + container.setId(containerId); + container.setNodeId(nodeId); + container.setNodeHttpAddress(nodeHttpAddress); + container.setResource(resource); + container.setPriority(priority); + container.setContainerToken(containerToken); + return container; + } + /** * Get the globally unique identifier for the container. * @return globally unique identifier for the container @@ -166,4 +184,41 @@ public static Container newInstance(ContainerId containerId, NodeId nodeId, @Private @Unstable public abstract void setContainerToken(Token containerToken); + + /** + *

+ * Get the meta-data from all auxiliary services running on the + * NodeManager. + *

+ *

+ * The meta-data is returned as a Map between the auxiliary service names and + * their corresponding per service meta-data as an opaque blob + * ByteBuffer + *

+ * + *

+ * To be able to interpret the per-service meta-data, you should consult the + * documentation for the Auxiliary-service configured on the NodeManager + *

+ * + * @return a Map between the names of auxiliary services and their + * corresponding meta-data + */ + @Public + @Stable + public abstract Map getAllServicesMetaData(); + + /** + * Set to the list of auxiliary services which have been started on the + * NodeManager. This is done only once when the + * NodeManager starts up + * + * @param allServicesMetaData + * A map from auxiliary service names to the opaque blob + * ByteBuffer for that auxiliary service + */ + @Private + @Unstable + public abstract void setAllServicesMetaData( + Map allServicesMetaData); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index a27cbb7..3aad36c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -74,6 +74,7 @@ message ContainerProto { optional ResourceProto resource = 4; optional PriorityProto priority = 5; optional hadoop.common.TokenProto container_token = 6; + repeated StringBytesMapProto services_meta_data = 7; } enum YarnApplicationStateProto { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java index b5f0be1..ad46907 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java @@ -205,6 +205,7 @@ private void addStartingContainer(StartedContainer startedContainer) parseAndThrowException(t); } allServiceResponse = response.getAllServicesMetaData(); + container.setAllServicesMetaData(allServiceResponse); startingContainer.state = ContainerState.RUNNING; } catch (YarnException e) { startingContainer.state = ContainerState.COMPLETE; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java index 76e87f5..8d8206d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java @@ -23,11 +23,14 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TreeSet; @@ -309,13 +312,15 @@ private void testContainerManagement(NMClientImpl nmClient, ContainerLaunchContext clc = Records.newRecord(ContainerLaunchContext.class); clc.setTokens(securityTokens); + Map serviceData; + Container spyContainer = spy(container); try { - nmClient.startContainer(container, clc); + serviceData = nmClient.startContainer(spyContainer, clc); } catch (YarnException e) { throw (AssertionError) (new AssertionError("Exception is not expected: " + e).initCause(e)); } - + verify(spyContainer).setAllServicesMetaData(serviceData); // leave one container unclosed if (++i < size) { // NodeManager may still need some time to make the container started diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java index 1700068..ec6e644 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java @@ -18,6 +18,12 @@ package org.apache.hadoop.yarn.api.records.impl.pb; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; @@ -33,6 +39,8 @@ import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; +import org.apache.hadoop.yarn.proto.YarnProtos.StringBytesMapProto; +import com.google.protobuf.ByteString; @Private @Unstable @@ -47,6 +55,7 @@ private Resource resource = null; private Priority priority = null; private Token containerToken = null; + private Map servicesMetaData = null; public ContainerPBImpl() { builder = ContainerProto.newBuilder(); @@ -106,6 +115,9 @@ private void mergeLocalToBuilder() { builder.getContainerToken())) { builder.setContainerToken(convertToProtoFormat(this.containerToken)); } + if (this.servicesMetaData != null) { + addServicesMetaDataToProto(); + } } private void mergeLocalToProto() { @@ -248,6 +260,35 @@ public void setContainerToken(Token containerToken) { this.containerToken = containerToken; } + @Override + public Map getAllServicesMetaData() { + initServicesMetaData(); + return this.servicesMetaData; + } + + @Override + public void setAllServicesMetaData(Map servicesMetaData) { + maybeInitBuilder(); + if (servicesMetaData == null) { + builder.clearServicesMetaData(); + } + this.servicesMetaData = servicesMetaData; + } + + private void initServicesMetaData() { + if (this.servicesMetaData != null) { + return; + } + ContainerProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getServicesMetaDataList(); + this.servicesMetaData = new HashMap(); + + for (StringBytesMapProto c : list) { + this.servicesMetaData.put(c.getKey(), + convertFromProtoFormat(c.getValue())); + } + } + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { return new ContainerIdPBImpl(p); } @@ -288,6 +329,51 @@ private TokenProto convertToProtoFormat(Token t) { return ((TokenPBImpl)t).getProto(); } + protected final ByteString convertToProtoFormat(ByteBuffer byteBuffer) { + return ProtoUtils.convertToProtoFormat(byteBuffer); + } + + protected final ByteBuffer convertFromProtoFormat(ByteString byteString) { + return ProtoUtils.convertFromProtoFormat(byteString); + } + + private void addServicesMetaDataToProto() { + maybeInitBuilder(); + builder.clearServicesMetaData(); + if (servicesMetaData == null) + return; + Iterable iterable = + new Iterable() { + + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator keyIter = servicesMetaData.keySet().iterator(); + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public StringBytesMapProto next() { + String key = keyIter.next(); + return StringBytesMapProto.newBuilder().setKey(key) + .setValue(convertToProtoFormat(servicesMetaData.get(key))) + .build(); + } + + @Override + public boolean hasNext() { + return keyIter.hasNext(); + } + }; + } + }; + builder.addAllServicesMetaData(iterable); + } + public String toString() { StringBuilder sb = new StringBuilder(); sb.append("Container: [");