diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java index d5bf03d89b1..6571a1c3832 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java @@ -30,17 +30,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse; -import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest; -import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse; -import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest; -import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse; -import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest; -import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse; -import org.apache.hadoop.yarn.api.protocolrecords.RestartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.RollbackResponse; +import org.apache.hadoop.yarn.api.protocolrecords.*; import org.junit.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -62,15 +52,6 @@ import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; -import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -516,5 +497,12 @@ public ContainerUpdateResponse updateContainer(ContainerUpdateRequest request) throws YarnException, IOException { return null; } + + @Override + public GetLocalizationStatusesResponse getLocalizationStatuses( + GetLocalizationStatusesRequest request) throws YarnException, + IOException { + return null; + } } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java index 0ae0380ea8a..5e96fb746ac 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java @@ -44,25 +44,7 @@ import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher.EventType; import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse; -import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest; -import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse; -import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest; -import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; -import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest; -import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse; -import org.apache.hadoop.yarn.api.protocolrecords.RestartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.RollbackResponse; -import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.*; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -521,6 +503,13 @@ public ContainerUpdateResponse updateContainer(ContainerUpdateRequest request) throws YarnException, IOException { return null; } + + @Override + public GetLocalizationStatusesResponse getLocalizationStatuses( + GetLocalizationStatusesRequest request) throws YarnException, + IOException { + return null; + } } @SuppressWarnings("serial") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java index 8fceb46e4c5..ec7813b0484 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java @@ -23,26 +23,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse; -import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest; -import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; -import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest; -import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse; -import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest; -import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationResponse; -import org.apache.hadoop.yarn.api.protocolrecords.RestartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.RollbackResponse; -import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.*; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; @@ -288,4 +269,21 @@ RollbackResponse rollbackLastReInitialization(ContainerId containerId) @Unstable CommitResponse commitLastReInitialization(ContainerId containerId) throws YarnException, IOException; + + /** + * API to request for the localization statuses of requested containers from + * the Node Manager. + * @param request {@link GetLocalizationStatusesRequest} which includes the + * container ids of all the containers whose localization + * statuses are needed. + * @return {@link GetLocalizationStatusesResponse} which contains the + * localization statuses of all the requested containers. + * @throws YarnException Exception specific to YARN. + * @throws IOException IOException thrown from the RPC layer. + */ + @Public + @Unstable + GetLocalizationStatusesResponse getLocalizationStatuses( + GetLocalizationStatusesRequest request) throws YarnException, + IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLocalizationStatusesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLocalizationStatusesRequest.java new file mode 100644 index 00000000000..cb39d57ee28 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLocalizationStatusesRequest.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.ContainerManagementProtocol; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.LocalizationStatus; +import org.apache.hadoop.yarn.util.Records; + +import java.util.List; + +/** + * The request sent by an application master to the node manager to get + * {@link LocalizationStatus}es of containers. + * + * @see ContainerManagementProtocol#getLocalizationStatuses( + * GetLocalizationStatusesRequest) + */ +@Public +@Unstable +public abstract class GetLocalizationStatusesRequest { + + @Public + @Unstable + public static GetLocalizationStatusesRequest newInstance( + List containerIds) { + GetLocalizationStatusesRequest request = + Records.newRecord(GetLocalizationStatusesRequest.class); + request.setContainerIds(containerIds); + return request; + } + + /** + * Get the list of container IDs of the containers for which the localization + * statuses are needed. + * + * @return the list of container IDs. + */ + @Public + @Unstable + public abstract List getContainerIds(); + + /** + * Sets the list of container IDs of containers for which the localization + * statuses are needed. + * @param containerIds the list of container IDs. + */ + @Public + @Unstable + public abstract void setContainerIds(List containerIds); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLocalizationStatusesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLocalizationStatusesResponse.java new file mode 100644 index 00000000000..fe151808c08 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetLocalizationStatusesResponse.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.ContainerManagementProtocol; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.LocalizationStatus; +import org.apache.hadoop.yarn.api.records.SerializedException; +import org.apache.hadoop.yarn.util.Records; + +import java.util.List; +import java.util.Map; + + +/** + * The response sent by the node manager to an application master when + * localization statuses are requested. + * + * @see ContainerManagementProtocol#getLocalizationStatuses( + * GetLocalizationStatusesRequest) + */ +@InterfaceAudience.Public +@InterfaceStability.Unstable +public abstract class GetLocalizationStatusesResponse { + + public static GetLocalizationStatusesResponse newInstance( + Map> statuses, + Map failedRequests) { + GetLocalizationStatusesResponse response = + Records.newRecord(GetLocalizationStatusesResponse.class); + response.setLocalizationStatuses(statuses); + return response; + } + + /** + * Get all the container localization statuses. + * + * @return container localization statuses. + */ + public abstract Map> getLocalizationStatuses(); + + /** + * Sets the container localization statuses. + * + * @param statuses container localization statuses. + */ + @InterfaceAudience.Private + public abstract void setLocalizationStatuses( + Map> statuses); + + + /** + * Get the containerId-to-exception map in which the exception indicates error + * from per container for failed requests. + * @return map of containerId-to-exception + */ + @InterfaceAudience.Private + public abstract Map getFailedRequests(); + + /** + * Set the containerId-to-exception map in which the exception indicates error + * from per container for failed request.s + */ + @InterfaceAudience.Private + public abstract void setFailedRequests( + Map failedContainers); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalizationState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalizationState.java new file mode 100644 index 00000000000..23520c3d41a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalizationState.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +/** + * State of localization. + */ +@Public +@Unstable +public enum LocalizationState { + + IN_PROGRESS, + + COMPLETED, + + FAILED +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalizationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalizationStatus.java new file mode 100644 index 00000000000..4bf469c1041 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalizationStatus.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +/** + * Represents the localization status of a resource. + *

+ * The status of the localization includes: + *

    + *
  • resource key
  • + *
  • {@code LocalizationState} of the resource
  • + *
+ *

+ */ +@Public +@Unstable +public abstract class LocalizationStatus { + + public static LocalizationStatus newInstance(String resourceKey, + LocalizationState localizationState) { + return newInstance(resourceKey, localizationState, null); + } + + public static LocalizationStatus newInstance(String resourceKey, + LocalizationState localizationState, + String diagnostics) { + LocalizationStatus status = Records.newRecord(LocalizationStatus.class); + status.setResourceKey(resourceKey); + status.setLocalizationState(localizationState); + status.setDiagnostics(diagnostics); + return status; + } + + /** + * Get the resource key. + * + * @return resource key. + */ + public abstract String getResourceKey(); + + /** + * Sets the resource key. + * @param resourceKey + */ + @InterfaceAudience.Private + public abstract void setResourceKey(String resourceKey); + + /** + * Get the localization sate. + * + * @return localization state. + */ + public abstract LocalizationState getLocalizationState(); + + /** + * Sets the localization state. + * @param state localization state + */ + @InterfaceAudience.Private + public abstract void setLocalizationState(LocalizationState state); + + /** + * Get the diagnostics. + * + * @return diagnostics. + */ + public abstract String getDiagnostics(); + + /** + * Sets the diagnostics. + * @param diagnostics diagnostics. + */ + @InterfaceAudience.Private + public abstract void setDiagnostics(String diagnostics); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto index 22b440693d3..1f8cafbd7da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto @@ -44,4 +44,8 @@ service ContainerManagementProtocolService { rpc restartContainer(ContainerIdProto) returns (RestartContainerResponseProto); rpc rollbackLastReInitialization(ContainerIdProto) returns (RollbackResponseProto); rpc commitLastReInitialization(ContainerIdProto) returns (CommitResponseProto); + + rpc getLocalizationStatuses(GetLocalizationStatusesRequestProto) + returns (GetLocalizationStatusesResponseProto); + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 248f775bdeb..48a95eaa9d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -543,3 +543,32 @@ message RunSharedCacheCleanerTaskRequestProto { message RunSharedCacheCleanerTaskResponseProto { optional bool accepted = 1; } + +// Localization +message GetLocalizationStatusesRequestProto { + repeated ContainerIdProto container_id = 1; +} + +message GetLocalizationStatusesResponseProto { + repeated ContainerLocalizationStatusesProto cntn_localization_statuses = 1; + repeated ContainerExceptionMapProto failed_requests = 2; +} + +enum LocalizationStateProto { + L_IN_PROGRESS = 1; + L_COMPLETED = 2; + L_FAILED = 3; +} + + +message LocalizationStatusProto { + optional string resource_key = 1; + optional LocalizationStateProto localization_state= 2; + optional string diagnostics = 3; +} + +message ContainerLocalizationStatusesProto { + optional ContainerIdProto container_id = 1; + repeated LocalizationStatusProto localization_statuses = 2; +} + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java index 7e471f34de3..af420210146 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java @@ -32,6 +32,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest; @@ -52,6 +54,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetLocalizationStatusesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetLocalizationStatusesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReInitializeContainerRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ReInitializeContainerResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ResourceLocalizationRequestPBImpl; @@ -74,6 +78,7 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ContainerUpdateRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetLocalizationStatusesRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ResourceLocalizationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SignalContainerRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersRequestProto; @@ -280,4 +285,21 @@ public CommitResponse commitLastReInitialization(ContainerId containerId) return null; } } + + @Override + public GetLocalizationStatusesResponse getLocalizationStatuses( + GetLocalizationStatusesRequest request) + throws YarnException, IOException { + GetLocalizationStatusesRequestProto requestProto = + ((GetLocalizationStatusesRequestPBImpl) request).getProto(); + try { + return new GetLocalizationStatusesResponsePBImpl( + proxy.getLocalizationStatuses(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java index 68e164582d6..ad8a75684b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse; import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest; import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerResponse; @@ -38,6 +39,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.CommitResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ContainerUpdateRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.ContainerUpdateResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetLocalizationStatusesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetLocalizationStatusesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.IncreaseContainersResourceRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.IncreaseContainersResourceResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesRequestPBImpl; @@ -65,6 +68,8 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetLocalizationStatusesRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetLocalizationStatusesResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReInitializeContainerRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ReInitializeContainerResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ResourceLocalizationRequestProto; @@ -264,4 +269,19 @@ public CommitResponseProto commitLastReInitialization( throw new ServiceException(e); } } + + @Override + public GetLocalizationStatusesResponseProto getLocalizationStatuses( + RpcController controller, GetLocalizationStatusesRequestProto request) + throws ServiceException { + GetLocalizationStatusesRequestPBImpl lclReq = + new GetLocalizationStatusesRequestPBImpl(request); + try { + GetLocalizationStatusesResponse response = real.getLocalizationStatuses( + lclReq); + return ((GetLocalizationStatusesResponsePBImpl)response).getProto(); + } catch (YarnException | IOException e) { + throw new ServiceException(e); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLocalizationStatusesRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLocalizationStatusesRequestPBImpl.java new file mode 100644 index 00000000000..f91c9eb0820 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLocalizationStatusesRequestPBImpl.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import com.google.protobuf.TextFormat; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetLocalizationStatusesRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetLocalizationStatusesRequestProtoOrBuilder; + +import java.util.ArrayList; +import java.util.List; + +@Private +@Unstable +public class GetLocalizationStatusesRequestPBImpl extends + GetLocalizationStatusesRequest { + GetLocalizationStatusesRequestProto proto = + GetLocalizationStatusesRequestProto.getDefaultInstance(); + GetLocalizationStatusesRequestProto.Builder builder; + boolean viaProto = false; + + private List containerIds; + + public GetLocalizationStatusesRequestPBImpl() { + builder = GetLocalizationStatusesRequestProto.newBuilder(); + } + + public GetLocalizationStatusesRequestPBImpl( + GetLocalizationStatusesRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetLocalizationStatusesRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (this.containerIds != null) { + addLocalContainerIdsToProto(); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetLocalizationStatusesRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void addLocalContainerIdsToProto() { + maybeInitBuilder(); + builder.clearContainerId(); + if (this.containerIds == null) + return; + List protoList = new ArrayList(); + for (ContainerId id : containerIds) { + protoList.add(convertToProtoFormat(id)); + } + builder.addAllContainerId(protoList); + } + + private void initLocalContainerIds() { + if (this.containerIds != null) { + return; + } + GetLocalizationStatusesRequestProtoOrBuilder p = viaProto ? proto : builder; + List containerIds = p.getContainerIdList(); + this.containerIds = new ArrayList(); + for (ContainerIdProto id : containerIds) { + this.containerIds.add(convertFromProtoFormat(id)); + } + } + + @Override + public List getContainerIds() { + initLocalContainerIds(); + return this.containerIds; + } + + @Override + public void setContainerIds(List containerIds) { + maybeInitBuilder(); + if (containerIds == null) + builder.clearContainerId(); + this.containerIds = containerIds; + } + + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { + return new ContainerIdPBImpl(p); + } + + private ContainerIdProto convertToProtoFormat(ContainerId t) { + return ((ContainerIdPBImpl) t).getProto(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLocalizationStatusesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLocalizationStatusesResponsePBImpl.java new file mode 100644 index 00000000000..3f29feb1a14 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetLocalizationStatusesResponsePBImpl.java @@ -0,0 +1,255 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import com.google.protobuf.TextFormat; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.LocalizationStatus; +import org.apache.hadoop.yarn.api.records.SerializedException; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.LocalizationStatusPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ContainerExceptionMapProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ContainerLocalizationStatusesProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetLocalizationStatusesResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetLocalizationStatusesResponseProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.LocalizationStatusProto; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Private +@Unstable +public class GetLocalizationStatusesResponsePBImpl extends + GetLocalizationStatusesResponse { + GetLocalizationStatusesResponseProto proto = + GetLocalizationStatusesResponseProto.getDefaultInstance(); + GetLocalizationStatusesResponseProto.Builder builder; + boolean viaProto = false; + + private Map> localizationStatuses; + private Map failedRequests; + + public GetLocalizationStatusesResponsePBImpl() { + builder = GetLocalizationStatusesResponseProto.newBuilder(); + } + + public GetLocalizationStatusesResponsePBImpl( + GetLocalizationStatusesResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetLocalizationStatusesResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void mergeLocalToBuilder() { + if (this.localizationStatuses != null) { + addLocalStatusesToProto(); + } + if (this.failedRequests != null) { + addFailedRequestsToProto(); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetLocalizationStatusesResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void addLocalStatusesToProto() { + maybeInitBuilder(); + builder.clearCntnLocalizationStatuses(); + if (this.localizationStatuses == null) { + return; + } + List protos = + new ArrayList(); + + this.localizationStatuses.forEach((containerId, statuses) -> { + if (statuses != null && !statuses.isEmpty()) { + ContainerLocalizationStatusesProto.Builder clProtoBuilder = + ContainerLocalizationStatusesProto.newBuilder(); + statuses.forEach(status -> { + clProtoBuilder.addLocalizationStatuses(convertToProtoFormat(status)); + }); + clProtoBuilder.setContainerId(convertToProtoFormat(containerId)); + protos.add(clProtoBuilder.build()); + } + }); + builder.addAllCntnLocalizationStatuses(protos); + } + + private void addFailedRequestsToProto() { + maybeInitBuilder(); + builder.clearFailedRequests(); + if (this.failedRequests == null) { + return; + } + List protoList = + new ArrayList(); + for (Map.Entry entry : this.failedRequests + .entrySet()) { + protoList.add(ContainerExceptionMapProto.newBuilder() + .setContainerId(convertToProtoFormat(entry.getKey())) + .setException(convertToProtoFormat(entry.getValue())).build()); + } + builder.addAllFailedRequests(protoList); + } + + + private void initLocalContainerStatuses() { + if (localizationStatuses != null) { + return; + } + GetLocalizationStatusesResponseProtoOrBuilder p = viaProto ? proto : + builder; + List protoList = + p.getCntnLocalizationStatusesList(); + localizationStatuses = new HashMap<>(); + + for (ContainerLocalizationStatusesProto clProto : protoList) { + List lsProtos = + clProto.getLocalizationStatusesList(); + + List statusesPerCntn = new ArrayList<>(); + lsProtos.forEach(lsProto -> { + statusesPerCntn.add(convertFromProtoFormat(lsProto)); + }); + + localizationStatuses.put(convertFromProtoFormat(clProto.getContainerId()), + statusesPerCntn); + } + } + + private void initFailedRequests() { + if (this.failedRequests != null) { + return; + } + GetLocalizationStatusesResponseProtoOrBuilder p = viaProto ? proto : + builder; + List protoList = p.getFailedRequestsList(); + this.failedRequests = new HashMap<>(); + for (ContainerExceptionMapProto ce : protoList) { + this.failedRequests.put(convertFromProtoFormat(ce.getContainerId()), + convertFromProtoFormat(ce.getException())); + } + } + + @Override + public Map> getLocalizationStatuses() { + initLocalContainerStatuses(); + return this.localizationStatuses; + } + + @Override + public void setLocalizationStatuses( + Map> statuses) { + maybeInitBuilder(); + if (statuses == null) { + builder.clearCntnLocalizationStatuses(); + } + this.localizationStatuses = statuses; + } + + @Override + public Map getFailedRequests() { + initFailedRequests(); + return this.failedRequests; + } + + @Override + public void setFailedRequests( + Map failedRequests) { + maybeInitBuilder(); + if (failedRequests == null) + builder.clearFailedRequests(); + this.failedRequests = failedRequests; + } + + private LocalizationStatusPBImpl convertFromProtoFormat( + LocalizationStatusProto p) { + return new LocalizationStatusPBImpl(p); + } + + private LocalizationStatusProto convertToProtoFormat( + LocalizationStatus t) { + return ((LocalizationStatusPBImpl) t).getProto(); + } + + + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { + return new ContainerIdPBImpl(p); + } + + private ContainerIdProto convertToProtoFormat(ContainerId t) { + return ((ContainerIdPBImpl) t).getProto(); + } + + private SerializedExceptionPBImpl convertFromProtoFormat( + SerializedExceptionProto p) { + return new SerializedExceptionPBImpl(p); + } + + private SerializedExceptionProto convertToProtoFormat(SerializedException t) { + return ((SerializedExceptionPBImpl) t).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalizationStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalizationStatusPBImpl.java new file mode 100644 index 00000000000..1feab85f48a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalizationStatusPBImpl.java @@ -0,0 +1,187 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.records.LocalizationState; +import org.apache.hadoop.yarn.api.records.LocalizationStatus; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.LocalizationStateProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.LocalizationStatusProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.LocalizationStatusProtoOrBuilder; + +@Private +@Unstable +public class LocalizationStatusPBImpl extends LocalizationStatus { + LocalizationStatusProto proto = LocalizationStatusProto.getDefaultInstance(); + LocalizationStatusProto.Builder builder; + boolean viaProto = false; + + private String resourceKey; + private LocalizationState localizationState; + private String diagnostics; + + public LocalizationStatusPBImpl() { + builder = LocalizationStatusProto.newBuilder(); + } + + public LocalizationStatusPBImpl(LocalizationStatusProto proto) { + this.proto = proto; + viaProto = true; + } + + public synchronized LocalizationStatusProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("LocalizationStatus: ["); + sb.append("ResourceKey: ").append(getResourceKey()).append(", "); + sb.append("LocalizationState: ").append(getLocalizationState()) + .append(", "); + sb.append("Diagnostics: ").append(getDiagnostics()).append(", "); + sb.append("]"); + return sb.toString(); + } + + private void mergeLocalToBuilder() { + if (resourceKey != null) { + builder.setResourceKey(this.resourceKey); + } + if (localizationState != null) { + builder.setLocalizationState(convertToProtoFormat(localizationState)); + } + if (diagnostics != null) { + builder.setDiagnostics(diagnostics); + } + } + + private synchronized void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private synchronized void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = LocalizationStatusProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public synchronized String getResourceKey() { + LocalizationStatusProtoOrBuilder p = viaProto ? proto : builder; + if (this.resourceKey != null) { + return this.resourceKey; + } + if (!p.hasResourceKey()) { + return null; + } + this.resourceKey = p.getResourceKey(); + return this.resourceKey; + } + + @Override + public synchronized void setResourceKey(String resourceKey) { + maybeInitBuilder(); + if (resourceKey == null) { + builder.clearResourceKey(); + } + this.resourceKey = resourceKey; + } + + @Override + public synchronized LocalizationState getLocalizationState() { + LocalizationStatusProtoOrBuilder p = viaProto ? proto : builder; + if (this.localizationState != null) { + return this.localizationState; + } + if (!p.hasLocalizationState()) { + return null; + } + this.localizationState = convertFromProtoFormat(p.getLocalizationState()); + return localizationState; + } + + @Override + public synchronized void setLocalizationState( + LocalizationState localizationState) { + maybeInitBuilder(); + if (localizationState == null) { + builder.clearLocalizationState(); + } + this.localizationState = localizationState; + } + + @Override + public synchronized String getDiagnostics() { + LocalizationStatusProtoOrBuilder p = viaProto ? proto : builder; + if (this.diagnostics != null) { + return this.diagnostics; + } + if (!p.hasDiagnostics()) { + return null; + } + this.diagnostics = p.getDiagnostics(); + return diagnostics; + } + + @Override + public synchronized void setDiagnostics(String diagnostics) { + maybeInitBuilder(); + if (diagnostics == null) { + builder.clearDiagnostics(); + } + this.diagnostics = diagnostics; + } + + private LocalizationStateProto convertToProtoFormat(LocalizationState e) { + return ProtoUtils.convertToProtoFormat(e); + } + + private LocalizationState convertFromProtoFormat(LocalizationStateProto e) { + return ProtoUtils.convertFromProtoFormat(e); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java index 4008a97f668..34237957de7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.LocalizationState; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; @@ -90,6 +91,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.NodeUpdateTypeProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos; import org.apache.hadoop.yarn.proto.YarnServiceProtos.ContainerUpdateTypeProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.LocalizationStateProto; import org.apache.hadoop.yarn.server.api.ContainerType; import com.google.protobuf.ByteString; @@ -578,6 +580,21 @@ public static ResourceTypes convertFromProtoFormat(ResourceTypesProto e) { TimedPlacementConstraintProto.DelayUnit u) { return TimedPlacementConstraint.DelayUnit.valueOf(u.name()); } + + //Localization State + private final static String LOCALIZATION_STATE_PREFIX = "L_"; + public static LocalizationStateProto convertToProtoFormat( + LocalizationState e) { + return LocalizationStateProto.valueOf(LOCALIZATION_STATE_PREFIX + e.name()); + } + + public static LocalizationState convertFromProtoFormat( + LocalizationStateProto e) { + return LocalizationState.valueOf(e.name() + .replace(LOCALIZATION_STATE_PREFIX, "")); + } + } + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java index dfe75349748..34e21984807 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java @@ -35,6 +35,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse; import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest; import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; @@ -245,5 +247,12 @@ public ContainerUpdateResponse updateContainer(ContainerUpdateRequest request) throws YarnException, IOException { return null; } + + @Override + public GetLocalizationStatusesResponse getLocalizationStatuses( + GetLocalizationStatusesRequest request) throws YarnException, + IOException { + return null; + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerResourceIncreaseRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerResourceIncreaseRPC.java index 6e9728475e4..1690b815d41 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerResourceIncreaseRPC.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerResourceIncreaseRPC.java @@ -31,6 +31,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest; @@ -227,5 +229,12 @@ public CommitResponse commitLastReInitialization(ContainerId containerId) throws YarnException, IOException { return null; } + + @Override + public GetLocalizationStatusesResponse getLocalizationStatuses( + GetLocalizationStatusesRequest request) + throws YarnException, IOException { + return null; + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java index 82dfaea32a9..dedabc07d6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java @@ -37,6 +37,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse; import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest; import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; @@ -420,6 +422,13 @@ public ContainerUpdateResponse updateContainer(ContainerUpdateRequest request) throws YarnException, IOException { return null; } + + @Override + public GetLocalizationStatusesResponse getLocalizationStatuses( + GetLocalizationStatusesRequest request) throws YarnException, + IOException { + return null; + } } public static ContainerTokenIdentifier newContainerTokenIdentifier( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 8a12c3c5105..fb47d818311 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -20,6 +20,9 @@ import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ByteString; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse; +import org.apache.hadoop.yarn.api.records.LocalizationStatus; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.UpdateContainerTokenEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerTokenUpdatedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.ContainerSchedulerEvent; @@ -1960,4 +1963,54 @@ public void handleCredentialUpdate() { dispatcher.getEventHandler().handle(new LogHandlerTokenUpdatedEvent()); } } + + @Override + public GetLocalizationStatusesResponse getLocalizationStatuses( + GetLocalizationStatusesRequest request) throws YarnException, + IOException { + Map> allStatuses = new HashMap<>(); + Map failedRequests = new HashMap<>(); + + UserGroupInformation remoteUgi = getRemoteUgi(); + NMTokenIdentifier identifier = selectNMTokenIdentifier(remoteUgi); + if (identifier == null) { + throw RPCUtil.getRemoteException(INVALID_NMTOKEN_MSG); + } + String remoteUser = remoteUgi.getUserName(); + for (ContainerId id : request.getContainerIds()) { + try { + List statuses = getLocalizationStatusesInternal(id, + identifier, remoteUser); + allStatuses.put(id, statuses); + } catch (YarnException e) { + failedRequests.put(id, SerializedException.newInstance(e)); + } + } + return GetLocalizationStatusesResponse.newInstance(allStatuses, + failedRequests); + } + + private List getLocalizationStatusesInternal( + ContainerId containerID, + NMTokenIdentifier nmTokenIdentifier, String remoteUser) + throws YarnException { + Container container = this.context.getContainers().get(containerID); + + LOG.info("Getting localization status for {}", containerID); + authorizeGetAndStopContainerRequest(containerID, container, false, + nmTokenIdentifier, remoteUser); + + String containerIDStr = containerID.toString(); + if (container == null) { + if (nodeStatusUpdater.isContainerRecentlyStopped(containerID)) { + throw RPCUtil.getRemoteException("Container " + containerIDStr + + " was recently stopped on node manager."); + } else { + throw RPCUtil.getRemoteException("Container " + containerIDStr + + " is not handled by this NodeManager"); + } + } + return container.getLocalizationStatuses(); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java index 4912d02758d..68504267b93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java @@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.LocalizationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.event.EventHandler; @@ -119,4 +120,10 @@ * @return true/false based on container's state */ boolean isContainerInFinalStates(); + + /** + * Get the localization statuses. + * @return localization statuses. + */ + List getLocalizationStatuses(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 6716dbb02e9..02cea5a89ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -36,6 +36,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.yarn.api.records.ContainerSubState; +import org.apache.hadoop.yarn.api.records.LocalizationStatus; import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.UpdateContainerSchedulerEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1445,7 +1446,8 @@ public void transition(ContainerImpl container, ContainerEvent event) { ContainerResourceFailedEvent failedEvent = (ContainerResourceFailedEvent) event; container.resourceSet - .resourceLocalizationFailed(failedEvent.getResource()); + .resourceLocalizationFailed(failedEvent.getResource(), + failedEvent.getDiagnosticMessage()); container.addDiagnostics(failedEvent.getDiagnosticMessage()); } } @@ -1461,7 +1463,7 @@ public void transition(ContainerImpl container, ContainerEvent event) { ContainerResourceFailedEvent failedEvent = (ContainerResourceFailedEvent) event; container.resourceSet.resourceLocalizationFailed( - failedEvent.getResource()); + failedEvent.getResource(), failedEvent.getDiagnosticMessage()); container.addDiagnostics("Container aborting re-initialization.. " + failedEvent.getDiagnosticMessage()); LOG.error("Container [" + container.getContainerId() + "] Re-init" + @@ -2255,4 +2257,14 @@ public boolean isContainerInFinalStates() { || state == ContainerState.EXITED_WITH_FAILURE || state == ContainerState.EXITED_WITH_SUCCESS; } + + @Override + public List getLocalizationStatuses() { + this.readLock.lock(); + try { + return resourceSet.getLocalizationStatuses(); + } finally { + this.readLock.unlock(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java index 745f8a88fb9..623d8a0b5e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java @@ -21,6 +21,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.LocalizationState; +import org.apache.hadoop.yarn.api.records.LocalizationStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,8 +50,8 @@ new ConcurrentHashMap<>(); private Map> pendingResources = new ConcurrentHashMap<>(); - private Set resourcesFailedToBeLocalized = - new HashSet<>(); + private final List resourcesFailedToBeLocalized = + new ArrayList<>(); // resources by visibility (public, private, app) private final List publicRsrcs = @@ -135,13 +137,18 @@ } } - public void resourceLocalizationFailed(LocalResourceRequest request) { + public void resourceLocalizationFailed(LocalResourceRequest request, + String diagnostics) { // Skip null request when localization failed for running container if (request == null) { return; } - pendingResources.remove(request); - resourcesFailedToBeLocalized.add(request); + Set keys = pendingResources.remove(request); + synchronized (resourcesFailedToBeLocalized) { + keys.forEach(key -> + resourcesFailedToBeLocalized.add(LocalizationStatus.newInstance(key, + LocalizationState.FAILED, diagnostics))); + } } public synchronized Map getLocalizationStatuses() { + List statuses = new ArrayList<>(); + localizedResources.forEach((key, path) -> { + LocalizationStatus status = LocalizationStatus.newInstance(key, + LocalizationState.COMPLETED); + statuses.add(status); + }); + + pendingResources.forEach((lrReq, keys) -> + keys.forEach(key -> { + LocalizationStatus status = LocalizationStatus.newInstance(key, + LocalizationState.IN_PROGRESS); + statuses.add(status); + })); + + synchronized (resourcesFailedToBeLocalized) { + statuses.addAll(resourcesFailedToBeLocalized); + } + return statuses; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java index 67dfef259df..81896ef7a7b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.LocalizationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.event.Dispatcher; @@ -258,4 +259,9 @@ public void sendPauseEvent(String description) { @Override public boolean isContainerInFinalStates() { return false; } + + @Override + public List getLocalizationStatuses() { + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java index ee974e3389f..440d971f208 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java @@ -28,6 +28,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse; import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest; import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReInitializeContainerRequest; @@ -370,4 +372,11 @@ public CommitResponse commitLastReInitialization(ContainerId containerId) throws YarnException, IOException { return null; } + + @Override + public GetLocalizationStatusesResponse getLocalizationStatuses( + GetLocalizationStatusesRequest request) throws YarnException, + IOException { + return null; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java index 1acf658b229..50e5865d3d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java @@ -44,6 +44,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse; import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest; import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; @@ -216,6 +218,13 @@ public CommitResponse commitLastReInitialization(ContainerId containerId) throws YarnException, IOException { return null; } + + @Override + public GetLocalizationStatusesResponse getLocalizationStatuses( + GetLocalizationStatusesRequest request) throws YarnException, + IOException { + return null; + } } public static class MockRMWithAMS extends MockRMWithCustomAMLauncher { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java index 03ccd764b6c..8fb1aa855dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java @@ -41,6 +41,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.CommitResponse; import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest; import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetLocalizationStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest; import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; @@ -212,6 +214,13 @@ public ContainerUpdateResponse updateContainer(ContainerUpdateRequest request) throws YarnException, IOException { return null; } + + @Override + public GetLocalizationStatusesResponse getLocalizationStatuses( + GetLocalizationStatusesRequest request) throws YarnException, + IOException { + return null; + } } @Test