diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java index d7c75f3..4e706bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java @@ -201,4 +201,18 @@ public void setHost(String host) { throw new UnsupportedOperationException( "subclass must implement this method"); } + + /** + * Get the Localization Status + * @return The LocalizationStatus + */ + + @Public + @Unstable + public abstract List getLocalizationStatus(); + + + @Private + @Unstable + public abstract void setLocalizationStatus(List status); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalizationState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalizationState.java new file mode 100644 index 0000000..4551029 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalizationState.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; + +/** + *

State of Localization Resource.

+ */ +@Public +@Stable +public enum LocalizationState { + /** LOCALIZATION PENDING */ + PENDING, + /** LOCALIZATION COMPLETED */ + COMPLETED, + /** LOCALIZATION FAILED */ + FAILED +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalizationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalizationStatus.java new file mode 100644 index 0000000..7317b79 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalizationStatus.java @@ -0,0 +1,78 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * {@code LocalizationStatus} represents the current status of + * {@code Localization Resource}. + *

+ * It provides details such as: + *

+ */ +@Public +@Stable +public abstract class LocalizationStatus { + + @Private + @Unstable + public static LocalizationStatus newInstance( + LocalResource localResource, LocalizationState localizationState) { + LocalizationStatus + localizationStatus = Records.newRecord(LocalizationStatus.class); + localizationStatus.setLocalizationState(localizationState); + localizationStatus.setLocalResource(localResource); + return localizationStatus; + } + + /** + * Get the LocalizationState of the Resource. + * @return LocalizationState of the Resource + */ + @Public + @Stable + public abstract LocalizationState getLocalizationState(); + + @Private + @Unstable + public abstract void setLocalizationState(LocalizationState state); + + + /** + * Get the LocalResource of the container. + * @return LocalResource of the container + */ + @Public + @Stable + public abstract LocalResource getLocalResource(); + + @Private + @Unstable + public abstract void setLocalResource(LocalResource localResource); + + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 4af5a97..876131a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -578,6 +578,18 @@ message ContainerStatusProto { optional ResourceProto capability = 5; optional ExecutionTypeProto executionType = 6 [default = GUARANTEED]; repeated StringStringMapProto container_attributes = 7; + repeated LocalizationStatusProto localization_status = 8; +} + +message LocalizationStatusProto { + optional LocalResourceProto resource = 1; + optional LocalizationStateProto state = 2; +} + +enum LocalizationStateProto { + LOCALIZATION_PENDING = 0; + LOCALIZATION_COMPLETED = 1; + LOCALIZATION_FAILED = 2 ; } enum ContainerExitStatusProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java index 219cf02..69c0ad7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.LocalizationStatus; import org.apache.hadoop.yarn.proto.YarnProtos; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; @@ -34,13 +35,18 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.LocalizationStatusProto; import java.util.Arrays; +import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; + + + @Private @Unstable public class ContainerStatusPBImpl extends ContainerStatus { @@ -52,6 +58,9 @@ private static final String HOST = "HOST"; private static final String IPS = "IPS"; private Map containerAttributes = new HashMap<>(); + private List localizationStatusList = + new ArrayList(); + public ContainerStatusPBImpl() { @@ -314,6 +323,65 @@ public synchronized void setHost(String host) { containerAttributes.put(HOST, host); } + @Override + public synchronized List getLocalizationStatus() { + ContainerStatusProtoOrBuilder p = viaProto ? proto : builder; + return convertFromProtoFormat(p.getLocalizationStatusList()); + } + + @Override + public void setLocalizationStatus(List status) { + maybeInitBuilder(); + if (status == null) { + builder.clearLocalizationStatus(); + return; + } + this.localizationStatusList = status; + + Iterable values = + new Iterable() { + + @Override public Iterator iterator() { + return new Iterator() { + private Iterator iterator = + localizationStatusList.iterator(); + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public LocalizationStatusProto next() { + LocalizationStatus stat = iterator.next(); + return convertToProtoFormat(stat); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + }; + this.builder.addAllLocalizationStatus(values); + } + + private LocalizationStatusProto convertToProtoFormat(LocalizationStatus e) { + return ProtoUtils.convertToProtoFormat(e); + } + + private List convertFromProtoFormat( + List e) { + List localizationStatusPBList = + new ArrayList(); + for (LocalizationStatusProto lp : e) { + localizationStatusPBList.add(new LocalizationStatusPBImpl(lp)); + + } + return localizationStatusPBList; + } + private ContainerStateProto convertToProtoFormat(ContainerState e) { return ProtoUtils.convertToProtoFormat(e); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalizationStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalizationStatusPBImpl.java new file mode 100644 index 0000000..f3d71d6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalizationStatusPBImpl.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalizationState; +import org.apache.hadoop.yarn.api.records.LocalizationStatus; +import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto; +import org.apache.hadoop.yarn.proto.YarnProtos.LocalizationStateProto; +import org.apache.hadoop.yarn.proto.YarnProtos.LocalizationStatusProto; +import org.apache.hadoop.yarn.proto.YarnProtos.LocalizationStatusProtoOrBuilder; + +@Private +@Unstable +public class LocalizationStatusPBImpl extends LocalizationStatus { + LocalizationStatusProto proto = LocalizationStatusProto.getDefaultInstance(); + LocalizationStatusProto.Builder builder = null; + boolean viaProto = false; + + private LocalResource localResource = null; + private LocalizationState localizationState = null; + + public LocalizationStatusPBImpl() { + builder = LocalizationStatusProto.newBuilder(); + } + + public LocalizationStatusPBImpl(LocalizationStatusProto proto) { + this.proto = proto; + viaProto = true; + } + + private synchronized void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = LocalizationStatusProto.newBuilder(proto); + } + viaProto = false; + } + + private LocalizationStateProto convertToProtoFormat(LocalizationState e) { + return ProtoUtils.convertToProtoFormat(e); + + } + + private LocalizationState convertFromProtoFormat(LocalizationStateProto e) { + return ProtoUtils.convertFromProtoFormat(e); + } + + private LocalResourcePBImpl convertFromProtoFormat(LocalResourceProto e) { + return new LocalResourcePBImpl(e); + } + + private LocalResourceProto convertToProtoFormat(LocalResource e) { + return ((LocalResourcePBImpl) e).getProto(); + } + + @Override + public synchronized LocalizationState getLocalizationState() { + LocalizationStatusProtoOrBuilder p = viaProto ? proto : builder; + if(this.localizationState != null) { + return this.localizationState; + } + if (!p.hasState()) { + return null; + } + return convertFromProtoFormat(p.getState()); + } + + @Override + public synchronized void setLocalizationState( + LocalizationState state) { + maybeInitBuilder(); + if (state == null) { + builder.clearState(); + return; + } + builder.setState(convertToProtoFormat(state)); + } + + @Override + public synchronized LocalResource getLocalResource() { + LocalizationStatusProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasResource()) { + return null; + } + this.localResource = convertFromProtoFormat(p.getResource()); + return this.localResource; + } + + @Override + public synchronized void setLocalResource(LocalResource localResource) { + + maybeInitBuilder(); + if (localResource == null) { + builder.clearResource(); + return; + } + builder.setResource(convertToProtoFormat(localResource)); + } + + private void mergeLocalToBuilder() { + if (localResource != null) { + builder.setResource(convertToProtoFormat(this.localResource)); + } + if (localizationState != null) { + //addContainerAttributesToProto(); + builder.setState(convertToProtoFormat(localizationState)); + } + } + + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + public synchronized LocalizationStatusProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java index 98bb979..e20ffae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -48,6 +48,8 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.records.LocalizationStatus; +import org.apache.hadoop.yarn.api.records.LocalizationState; import org.apache.hadoop.yarn.proto.YarnProtos; import org.apache.hadoop.yarn.proto.YarnProtos.AMCommandProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAccessTypeProto; @@ -73,6 +75,9 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ContainerUpdateTypeProto; +import org.apache.hadoop.yarn.proto.YarnProtos.LocalizationStatusProto; +import org.apache.hadoop.yarn.proto.YarnProtos.LocalizationStateProto; + import org.apache.hadoop.yarn.server.api.ContainerType; import com.google.protobuf.ByteString; @@ -94,6 +99,35 @@ public static ContainerState convertFromProtoFormat(ContainerStateProto e) { } /* + * LocalizationStatus + */ + public static LocalizationStatusProto convertToProtoFormat( + LocalizationStatus e) { + return ((LocalizationStatusPBImpl) e).getProto(); + } + + public static LocalizationStatus convertFromProtoFormat( + LocalizationStatusProto e) { + return new LocalizationStatusPBImpl(e); + } + + /* + * LocalizationState + */ + private static String LOCALIZATION_STATE_PREFIX = "LOCALIZATION_"; + + public static LocalizationStateProto convertToProtoFormat( + LocalizationState e) { + return LocalizationStateProto.valueOf(LOCALIZATION_STATE_PREFIX + e.name()); + } + + public static LocalizationState convertFromProtoFormat( + LocalizationStateProto e) { + return LocalizationState + .valueOf(e.name().replace(LOCALIZATION_STATE_PREFIX, "")); + } + + /* * NodeState */ private static String NODE_STATE_PREFIX = "NS_"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java