diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java index 28508a9..d0e66ea 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java @@ -20,7 +20,9 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; @@ -44,10 +46,13 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy; import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -139,13 +144,18 @@ public synchronized void launch(ContainerRemoteLaunchEvent event) { event.getContainerLaunchContext(); // Now launch the actual container - StartContainerRequest startRequest = Records - .newRecord(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainerToken(event.getContainerToken()); - StartContainerResponse response = - proxy.getContainerManagementProtocol().startContainer(startRequest); - + StartContainerRequest startRequest = + StartContainerRequest.newInstance(containerLaunchContext, + event.getContainerToken()); + List list = new ArrayList(); + list.add(startRequest); + StartContainersRequest requestList = StartContainersRequest.newInstance(list); + StartContainersResponse response = + proxy.getContainerManagementProtocol().startContainers(requestList); + if (response.getFailedRequests() != null + && response.getFailedRequests().containsKey(containerID)) { + throw response.getFailedRequests().get(containerID).deSerialize(); + } ByteBuffer portInfo = response.getAllServicesMetaData().get( ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID); @@ -192,13 +202,17 @@ public synchronized void kill() { proxy = getCMProxy(this.containerMgrAddress, this.containerID); // kill the remote container if already launched - StopContainerRequest stopRequest = Records - .newRecord(StopContainerRequest.class); - stopRequest.setContainerId(this.containerID); - proxy.getContainerManagementProtocol().stopContainer(stopRequest); - + List ids = new ArrayList(); + ids.add(this.containerID); + StopContainersRequest request = StopContainersRequest.newInstance(ids); + StopContainersResponse response = + proxy.getContainerManagementProtocol().stopContainers(request); + if (response.getFailedRequests() != null + && response.getFailedRequests().containsKey(this.containerID)) { + throw response.getFailedRequests().get(this.containerID) + .deSerialize(); + } } catch (Throwable t) { - // ignore the cleanup failure String message = "cleanup failed for container " + this.containerID + " : " diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java index 563c31b..dd1060c 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java @@ -24,6 +24,8 @@ import java.lang.reflect.UndeclaredThrowableException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; @@ -52,12 +54,13 @@ import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -393,18 +396,18 @@ public ContainerManagementProtocolProxyData getCMProxy( private ContainerStatus status = null; @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws IOException { - GetContainerStatusResponse response = recordFactory - .newRecordInstance(GetContainerStatusResponse.class); - response.setStatus(status); - return response; + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws IOException { + List statuses = new ArrayList(); + statuses.add(status); + return GetContainerStatusesResponse.newInstance(statuses, null); } @Override - public StartContainerResponse startContainer(StartContainerRequest request) + public StartContainersResponse startContainers(StartContainersRequest requests) throws IOException { + StartContainerRequest request = requests.getStartContainerRequests().get(0); ContainerTokenIdentifier containerTokenIdentifier = MRApp.newContainerTokenIdentifier(request.getContainerToken()); @@ -412,8 +415,8 @@ public StartContainerResponse startContainer(StartContainerRequest request) Assert.assertEquals(MRApp.NM_HOST + ":" + MRApp.NM_PORT, containerTokenIdentifier.getNmHostAddress()); - StartContainerResponse response = recordFactory - .newRecordInstance(StartContainerResponse.class); + StartContainersResponse response = recordFactory + .newRecordInstance(StartContainersResponse.class); status = recordFactory.newRecordInstance(ContainerStatus.class); try { // make the thread sleep to look like its not going to respond @@ -429,7 +432,7 @@ public StartContainerResponse startContainer(StartContainerRequest request) } @Override - public StopContainerResponse stopContainer(StopContainerRequest request) + public StopContainersResponse stopContainers(StopContainersRequest request) throws IOException { Exception e = new Exception("Dummy function", new Exception( "Dummy function cause")); diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java index dc6ca6a..6f21c87 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java @@ -45,12 +45,12 @@ import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher.EventType; import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -162,8 +162,8 @@ public void testHandle() throws Exception { try { ContainerId contId = makeContainerId(0l, 0, 0, 1); TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0); - StartContainerResponse startResp = - recordFactory.newRecordInstance(StartContainerResponse.class); + StartContainersResponse startResp = + recordFactory.newRecordInstance(StartContainersResponse.class); startResp.setAllServicesMetaData(serviceResponse); @@ -176,14 +176,14 @@ public void testHandle() throws Exception { .thenReturn(contId); when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId); when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress); - when(mockCM.startContainer(any(StartContainerRequest.class))).thenReturn(startResp); + when(mockCM.startContainers(any(StartContainersRequest.class))).thenReturn(startResp); when(mockLaunchEvent.getContainerToken()).thenReturn( createNewContainerToken(contId, cmAddress)); ut.handle(mockLaunchEvent); ut.waitForPoolToIdle(); - verify(mockCM).startContainer(any(StartContainerRequest.class)); + verify(mockCM).startContainers(any(StartContainersRequest.class)); LOG.info("inserting cleanup event"); ContainerLauncherEvent mockCleanupEvent = @@ -198,7 +198,7 @@ public void testHandle() throws Exception { ut.waitForPoolToIdle(); - verify(mockCM).stopContainer(any(StopContainerRequest.class)); + verify(mockCM).stopContainers(any(StopContainersRequest.class)); } finally { ut.stop(); } @@ -224,8 +224,8 @@ public void testOutOfOrder() throws Exception { ContainerId contId = makeContainerId(0l, 0, 0, 1); TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0); String cmAddress = "127.0.0.1:8000"; - StartContainerResponse startResp = - recordFactory.newRecordInstance(StartContainerResponse.class); + StartContainersResponse startResp = + recordFactory.newRecordInstance(StartContainersResponse.class); startResp.setAllServicesMetaData(serviceResponse); LOG.info("inserting cleanup event"); @@ -241,7 +241,7 @@ public void testOutOfOrder() throws Exception { ut.waitForPoolToIdle(); - verify(mockCM, never()).stopContainer(any(StopContainerRequest.class)); + verify(mockCM, never()).stopContainers(any(StopContainersRequest.class)); LOG.info("inserting launch event"); ContainerRemoteLaunchEvent mockLaunchEvent = @@ -252,14 +252,14 @@ public void testOutOfOrder() throws Exception { .thenReturn(contId); when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId); when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress); - when(mockCM.startContainer(any(StartContainerRequest.class))).thenReturn(startResp); + when(mockCM.startContainers(any(StartContainersRequest.class))).thenReturn(startResp); when(mockLaunchEvent.getContainerToken()).thenReturn( createNewContainerToken(contId, cmAddress)); ut.handle(mockLaunchEvent); ut.waitForPoolToIdle(); - verify(mockCM, never()).startContainer(any(StartContainerRequest.class)); + verify(mockCM, never()).startContainers(any(StartContainersRequest.class)); } finally { ut.stop(); } @@ -286,8 +286,8 @@ public void testMyShutdown() throws Exception { ContainerId contId = makeContainerId(0l, 0, 0, 1); TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0); String cmAddress = "127.0.0.1:8000"; - StartContainerResponse startResp = - recordFactory.newRecordInstance(StartContainerResponse.class); + StartContainersResponse startResp = + recordFactory.newRecordInstance(StartContainersResponse.class); startResp.setAllServicesMetaData(serviceResponse); LOG.info("inserting launch event"); @@ -299,20 +299,20 @@ public void testMyShutdown() throws Exception { .thenReturn(contId); when(mockLaunchEvent.getTaskAttemptID()).thenReturn(taskAttemptId); when(mockLaunchEvent.getContainerMgrAddress()).thenReturn(cmAddress); - when(mockCM.startContainer(any(StartContainerRequest.class))).thenReturn(startResp); + when(mockCM.startContainers(any(StartContainersRequest.class))).thenReturn(startResp); when(mockLaunchEvent.getContainerToken()).thenReturn( createNewContainerToken(contId, cmAddress)); ut.handle(mockLaunchEvent); ut.waitForPoolToIdle(); - verify(mockCM).startContainer(any(StartContainerRequest.class)); + verify(mockCM).startContainers(any(StartContainersRequest.class)); // skip cleanup and make sure stop kills the container } finally { ut.stop(); - verify(mockCM).stopContainer(any(StopContainerRequest.class)); + verify(mockCM).stopContainers(any(StopContainersRequest.class)); } } @@ -341,8 +341,8 @@ public void testContainerCleaned() throws Exception { ContainerId contId = makeContainerId(0l, 0, 0, 1); TaskAttemptId taskAttemptId = makeTaskAttemptId(0l, 0, 0, TaskType.MAP, 0); String cmAddress = "127.0.0.1:8000"; - StartContainerResponse startResp = - recordFactory.newRecordInstance(StartContainerResponse.class); + StartContainersResponse startResp = + recordFactory.newRecordInstance(StartContainersResponse.class); startResp.setAllServicesMetaData(serviceResponse); @@ -415,7 +415,7 @@ private Token createNewContainerToken(ContainerId contId, this.completeLaunchBarrier = completeLaunchBarrier; } @Override - public StartContainerResponse startContainer(StartContainerRequest request) + public StartContainersResponse startContainers(StartContainersRequest request) throws IOException { try { startLaunchBarrier.await(); @@ -433,16 +433,14 @@ public StartContainerResponse startContainer(StartContainerRequest request) } @Override - public StopContainerResponse stopContainer(StopContainerRequest request) + public StopContainersResponse stopContainers(StopContainersRequest request) throws IOException { - return null; } @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws IOException { - + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws IOException { return null; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java index de330c5..7fc77ea 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ContainerManagementProtocol.java @@ -22,17 +22,17 @@ import org.apache.hadoop.classification.InterfaceAudience.Public; import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.exceptions.InvalidContainerException; import org.apache.hadoop.yarn.exceptions.NMNotYetReadyException; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -50,10 +50,12 @@ @Public @Stable public interface ContainerManagementProtocol { + /** *

- * The ApplicationMaster requests a NodeManager to - * start a {@link Container} allocated to it using this interface. + * The ApplicationMaster provides a list of + * {@link StartContainerRequest}s to a NodeManager to + * start {@link Container}s allocated to it using this interface. *

* *

@@ -65,82 +67,101 @@ *

* *

- * Currently the NodeManager sends an immediate, empty response - * via {@link StartContainerResponse} to signify acceptance of the request and - * throws an exception in case of errors. The ApplicationMaster - * can use {@link #getContainerStatus(GetContainerStatusRequest)} to get - * updated status of the to-be-launched or launched container. + * The NodeManager sends a response via + * {@link StartContainersResponse} which includes a list of + * {@link Container}s of successfully launched {@link Container}s, a + * containerId-to-exception map for each failed {@link StartContainerRequest} in + * which the exception indicates errors from per container and a + * allServicesMetaData map between the names of auxiliary services and their + * corresponding meta-data. Note: None-container-specific exceptions will + * still be thrown by the API method itself. + *

+ *

+ * The ApplicationMaster can use + * {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated + * statuses of the to-be-launched or launched containers. *

* * @param request - * request to start a container - * @return empty response to indicate acceptance of the request or an - * exception + * request to start a list of containers + * @return response including conatinerIds of all successfully launched + * containers, a containerId-to-exception map for failed requests and + * a allServicesMetaData map. * @throws YarnException * @throws IOException * @throws NMNotYetReadyException * This exception is thrown when NM starts from scratch but has not * yet connected with RM. - * @throws InvalidContainerException - * This exception is thrown when NM is rejecting start-container - * requests for containers allocated by a previous instance of the - * RM */ - @Public - @Stable - StartContainerResponse startContainer(StartContainerRequest request) + StartContainersResponse startContainers(StartContainersRequest request) throws YarnException, IOException; /** - *

The ApplicationMaster requests a NodeManager - * to stop a {@link Container} allocated to it using this interface. + *

+ * The ApplicationMaster requests a NodeManager to + * stop a list of {@link Container}s allocated to it using this + * interface. *

* - *

The ApplicationMaster sends a - * {@link StopContainerRequest} which includes the {@link ContainerId} of the - * container to be stopped.

+ *

+ * The ApplicationMaster sends a {@link StopContainersRequest} + * which includes the {@link ContainerId}s of the containers to be stopped. + *

* - *

Currently the NodeManager sends an immediate, empty - * response via {@link StopContainerResponse} to signify acceptance of the - * request and throws an exception in case of errors. The - * ApplicationMaster can use - * {@link #getContainerStatus(GetContainerStatusRequest)} to get updated - * status of the container.

+ *

+ * The NodeManager sends a response via + * {@link StopContainersResponse} which includes a list of {@link ContainerId} + * s of successfully stopped containers, a containerId-to-exception map for + * each failed request in which the exception indicates errors from per + * container. Note: None-container-specific exceptions will still be thrown by + * the API method itself. ApplicationMaster can use + * {@link #getContainerStatuses(GetContainerStatusesRequest)} to get updated + * statuses of the containers. + *

* - * @param request request to stop a container - * @return empty response to indicate acceptance of the request - * or an exception + * @param request + * request to stop a list of containers + * @return response which includes a list of containerIds of successfully + * stopped containers, a containerId-to-exception map for failed + * requests. * @throws YarnException * @throws IOException */ - @Public - @Stable - StopContainerResponse stopContainer(StopContainerRequest request) + StopContainersResponse stopContainers(StopContainersRequest request) throws YarnException, IOException; /** - *

The api used by the ApplicationMaster to request for - * current status of a Container from the - * NodeManager.

+ *

+ * The API used by the ApplicationMaster to request for current + * statuses of Containers from the NodeManager. + *

+ * + *

+ * The ApplicationMaster sends a + * {@link GetContainerStatusesRequest} which includes the {@link ContainerId}s + * of all containers whose statuses are needed. + *

+ * + *

+ * The NodeManager responds with + * {@link GetContainerStatusesResponse} which includes a list of + * {@link ContainerStatus} of the successfully queried containers and a + * containerId-to-exception map for each failed request in which the exception + * indicates errors from per container. Note: None-container-specific + * exceptions will still be thrown by the API method itself. + *

+ * + * @param request + * request to get ContainerStatuses of containers with + * the specified ContainerIds + * @return response containing the list of ContainerStatus of the + * successfully queried containers and a containerId-to-exception map + * for failed requests. * - *

The ApplicationMaster sends a - * {@link GetContainerStatusRequest} which includes the {@link ContainerId} of - * the container whose status is needed.

- * - *

The NodeManager responds with - *{@link GetContainerStatusResponse} which includes the - *{@link ContainerStatus} of the container.

- * - * @param request request to get ContainerStatus of a container - * with the specified ContainerId - * @return response containing the ContainerStatus of the - * container * @throws YarnException * @throws IOException */ - @Public - @Stable - GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws YarnException, + GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException, IOException; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusRequest.java deleted file mode 100644 index 6d0f81e..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusRequest.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.protocolrecords; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.util.Records; - -/** - *

The request sent by the ApplicationMaster to the - * NodeManager to get {@link ContainerStatus} of a container.

- * - * @see ContainerManagementProtocol#getContainerStatus(GetContainerStatusRequest) - */ -@Public -@Stable -public abstract class GetContainerStatusRequest { - - @Public - @Stable - public static GetContainerStatusRequest newInstance(ContainerId containerId) { - GetContainerStatusRequest request = - Records.newRecord(GetContainerStatusRequest.class); - request.setContainerId(containerId); - return request; - } - - /** - * Get the ContainerId of container for which to obtain the - * ContainerStatus. - * @return ContainerId of container for which to obtain the - * ContainerStatus - */ - @Public - @Stable - public abstract ContainerId getContainerId(); - - /** - * Set the ContainerId of container for which to obtain the - * ContainerStatus - * @param containerId ContainerId of container for which to - * obtain the ContainerStatus - */ - @Public - @Stable - public abstract void setContainerId(ContainerId containerId); -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusResponse.java deleted file mode 100644 index acf70b3..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusResponse.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.protocolrecords; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.util.Records; - -/** - *

The response sent by the NodeManager to the - * ApplicationMaster when asked to obtain the status - * of a container.

- * - * @see ContainerManagementProtocol#getContainerStatus(GetContainerStatusRequest) - */ -@Public -@Stable -public abstract class GetContainerStatusResponse { - - @Private - @Unstable - public static GetContainerStatusResponse newInstance( - ContainerStatus containerStatus) { - GetContainerStatusResponse response = - Records.newRecord(GetContainerStatusResponse.class); - response.setStatus(containerStatus); - return response; - } - - /** - * Get the ContainerStatus of the container. - * @return ContainerStatus of the container - */ - @Public - @Stable - public abstract ContainerStatus getStatus(); - - @Private - @Unstable - public abstract void setStatus(ContainerStatus containerStatus); -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusesRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusesRequest.java new file mode 100644 index 0000000..f9f77a3 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusesRequest.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.api.ContainerManagementProtocol; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * The request sent by the ApplicationMaster to the + * NodeManager to get {@link ContainerStatus} of requested + * containers. + *

+ * + * @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest) + */ +@Public +@Stable +public abstract class GetContainerStatusesRequest { + + @Public + @Stable + public static GetContainerStatusesRequest newInstance( + List containerIds) { + GetContainerStatusesRequest request = + Records.newRecord(GetContainerStatusesRequest.class); + request.setContainerIds(containerIds); + return request; + } + + /** + * Get the list of ContainerIds of containers for which to obtain + * the ContainerStatus. + * + * @return the list of ContainerIds of containers for which to + * obtain the ContainerStatus. + */ + @Public + @Stable + public abstract List getContainerIds(); + + /** + * Set a list of ContainerIds of containers for which to obtain + * the ContainerStatus + * + * @param containerIds + * a list of ContainerIds of containers for which to + * obtain the ContainerStatus + */ + @Public + @Stable + public abstract void setContainerIds(List containerIds); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusesResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusesResponse.java new file mode 100644 index 0000000..b0a0f0e --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetContainerStatusesResponse.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.ContainerManagementProtocol; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.SerializedException; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * The response sent by the NodeManager to the + * ApplicationMaster when asked to obtain the + * ContainerStatus of requested containers. + *

+ * + * @see ContainerManagementProtocol#getContainerStatuses(GetContainerStatusesRequest) + */ +@Public +@Stable +public abstract class GetContainerStatusesResponse { + + @Private + @Unstable + public static GetContainerStatusesResponse newInstance( + List statuses, + Map failedRequests) { + GetContainerStatusesResponse response = + Records.newRecord(GetContainerStatusesResponse.class); + response.setContainerStatuses(statuses); + response.setFailedRequests(failedRequests); + return response; + } + + /** + * Get the ContainerStatuses of the requested containers. + * + * @return ContainerStatuses of the requested containers. + */ + @Public + @Stable + public abstract List getContainerStatuses(); + + /** + * Set the ContainerStatuses of the requested containers. + */ + @Private + @Unstable + public abstract void setContainerStatuses(List statuses); + + /** + * Get the containerId-to-exception map in which the exception indicates error + * from per container for failed requests + */ + @Public + @Stable + public abstract Map getFailedRequests(); + + /** + * Set the containerId-to-exception map in which the exception indicates error + * from per container for failed requests + */ + @Private + @Unstable + public abstract void setFailedRequests( + Map failedContainers); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerResponse.java deleted file mode 100644 index 8df11db..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerResponse.java +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.protocolrecords; - -import java.nio.ByteBuffer; -import java.util.Map; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.util.Records; - -/** - *

The response sent by the NodeManager to the - * ApplicationMaster when asked to start an - * allocated container.

- * - * @see ContainerManagementProtocol#startContainer(StartContainerRequest) - */ -@Public -@Stable -public abstract class StartContainerResponse { - - @Private - @Unstable - public static StartContainerResponse newInstance( - Map servicesMetaData) { - StartContainerResponse response = - Records.newRecord(StartContainerResponse.class); - response.setAllServicesMetaData(servicesMetaData); - return response; - } - - /** - *

- * Get the meta-data from all auxiliary services running on the - * NodeManager. - *

- *

- * The meta-data is returned as a Map between the auxiliary service names and - * their corresponding per service meta-data as an opaque blob - * ByteBuffer - *

- * - *

- * To be able to interpret the per-service meta-data, you should consult the - * documentation for the Auxiliary-service configured on the NodeManager - *

- * - * @return a Map between the names of auxiliary services and their - * corresponding meta-data - */ - @Public - @Stable - public abstract Map getAllServicesMetaData(); - - /** - * Set to the list of auxiliary services which have been started on the - * NodeManager. This is done only once when the - * NodeManager starts up - * @param allServicesMetaData A map from auxiliary service names to the opaque - * blob ByteBuffer for that auxiliary service - */ - @Private - @Unstable - public abstract void setAllServicesMetaData( - Map allServicesMetaData); -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainersRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainersRequest.java new file mode 100644 index 0000000..1f5b51c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainersRequest.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.api.ContainerManagementProtocol; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * The request which contains a list of {@link StartContainerRequest} sent by + * the ApplicationMaster to the NodeManager to + * start containers. + *

+ * + *

+ * In each {@link StartContainerRequest}, the ApplicationMaster has + * to provide details such as allocated resource capability, security tokens (if + * enabled), command to be executed to start the container, environment for the + * process, necessary binaries/jar/shared-objects etc. via the + * {@link ContainerLaunchContext}. + *

+ * + * @see ContainerManagementProtocol#startContainers(StartContainersRequest) + */ +@Public +@Stable +public abstract class StartContainersRequest { + + @Public + @Stable + public static StartContainersRequest newInstance( + List requests) { + StartContainersRequest request = + Records.newRecord(StartContainersRequest.class); + request.setStartContainerRequests(requests); + return request; + } + + /** + * Get a list of {@link StartContainerRequest} to start containers. + * @return a list of {@link StartContainerRequest} to start containers. + */ + @Public + @Stable + public abstract List getStartContainerRequests(); + + /** + * Set a list of {@link StartContainerRequest} to start containers. + * @param request a list of {@link StartContainerRequest} to start containers + */ + @Public + @Stable + public abstract void setStartContainerRequests( + List request); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainersResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainersResponse.java new file mode 100644 index 0000000..b63bf7d --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainersResponse.java @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.ContainerManagementProtocol; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.SerializedException; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * The response sent by the NodeManager to the + * ApplicationMaster when asked to start an allocated + * container. + *

+ * + * @see ContainerManagementProtocol#startContainer(StartContainerRequest) + */ +@Public +@Stable +public abstract class StartContainersResponse { + + @Private + @Unstable + public static StartContainersResponse newInstance( + Map servicesMetaData, + List succeededContainers, + Map failedContainers) { + StartContainersResponse response = + Records.newRecord(StartContainersResponse.class); + response.setAllServicesMetaData(servicesMetaData); + response.setSuccessfullyStartedContainers(succeededContainers); + response.setFailedRequests(failedContainers); + return response; + } + + /** + * Get the list of ContainerId s of the containers that are + * started successfully. + * + * @return the list of ContainerId s of the containers that are + * started successfully. + * @see ContainerManagementProtocol#startContainer(StartContainerRequest) + */ + @Public + @Stable + public abstract List getSuccessfullyStartedContainers(); + + @Private + @Unstable + public abstract void setSuccessfullyStartedContainers( + List succeededContainers); + + /** + * Get the containerId-to-exception map in which the exception indicates error + * from per container for failed requests + */ + @Public + @Stable + public abstract Map getFailedRequests(); + + /** + * Set the containerId-to-exception map in which the exception indicates error + * from per container for failed requests + */ + @Private + @Unstable + public abstract void setFailedRequests( + Map failedContainers); + + /** + *

+ * Get the meta-data from all auxiliary services running on the + * NodeManager. + *

+ *

+ * The meta-data is returned as a Map between the auxiliary service names and + * their corresponding per service meta-data as an opaque blob + * ByteBuffer + *

+ * + *

+ * To be able to interpret the per-service meta-data, you should consult the + * documentation for the Auxiliary-service configured on the NodeManager + *

+ * + * @return a Map between the names of auxiliary services and their + * corresponding meta-data + */ + @Public + @Stable + public abstract Map getAllServicesMetaData(); + + /** + * Set to the list of auxiliary services which have been started on the + * NodeManager. This is done only once when the + * NodeManager starts up + * + * @param allServicesMetaData + * A map from auxiliary service names to the opaque blob + * ByteBuffer for that auxiliary service + */ + @Private + @Unstable + public abstract void setAllServicesMetaData( + Map allServicesMetaData); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainerRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainerRequest.java deleted file mode 100644 index 1d4a211..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainerRequest.java +++ /dev/null @@ -1,61 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn.api.protocolrecords; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.util.Records; - -/** - *

The request sent by the ApplicationMaster to the - * NodeManager to stop a container.

- * - * @see ContainerManagementProtocol#stopContainer(StopContainerRequest) - */ -@Public -@Stable -public abstract class StopContainerRequest { - - @Public - @Stable - public static StopContainerRequest newInstance(ContainerId containerId) { - StopContainerRequest request = - Records.newRecord(StopContainerRequest.class); - request.setContainerId(containerId); - return request; - } - - /** - * Get the ContainerId of the container to be stopped. - * @return ContainerId of container to be stopped - */ - @Public - @Stable - public abstract ContainerId getContainerId(); - - /** - * Set the ContainerId of the container to be stopped. - * @param containerId ContainerId of the container to be stopped - */ - @Public - @Stable - public abstract void setContainerId(ContainerId containerId); -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainerResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainerResponse.java deleted file mode 100644 index c8633f9..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainerResponse.java +++ /dev/null @@ -1,47 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn.api.protocolrecords; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.util.Records; - -/** - *

The response sent by the NodeManager to the - * ApplicationMaster when asked to stop an - * allocated container.

- * - *

Currently, this is empty.

- * - * @see ContainerManagementProtocol#stopContainer(StopContainerRequest) - */ -@Public -@Stable -public abstract class StopContainerResponse { - @Private - @Unstable - public static StopContainerResponse newInstance() { - StopContainerResponse response = - Records.newRecord(StopContainerResponse.class); - return response; - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainersRequest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainersRequest.java new file mode 100644 index 0000000..8ea186c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainersRequest.java @@ -0,0 +1,63 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.api.ContainerManagementProtocol; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.util.Records; + +/** + *

The request sent by the ApplicationMaster to the + * NodeManager to stop containers.

+ * + * @see ContainerManagementProtocol#stopContainers(StopContainersRequest) + */ +@Public +@Stable +public abstract class StopContainersRequest { + + @Public + @Stable + public static StopContainersRequest newInstance(List containerIds) { + StopContainersRequest request = + Records.newRecord(StopContainersRequest.class); + request.setContainerIds(containerIds); + return request; + } + + /** + * Get the ContainerIds of the containers to be stopped. + * @return ContainerIds of containers to be stopped + */ + @Public + @Stable + public abstract List getContainerIds(); + + /** + * Set the ContainerIds of the containers to be stopped. + * @param containerIds ContainerIds of the containers to be stopped + */ + @Public + @Stable + public abstract void setContainerIds(List containerIds); +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainersResponse.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainersResponse.java new file mode 100644 index 0000000..ee83ca1 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StopContainersResponse.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.ContainerManagementProtocol; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.SerializedException; +import org.apache.hadoop.yarn.util.Records; + +/** + *

+ * The response sent by the NodeManager to the + * ApplicationMaster when asked to stop an allocated + * container. + *

+ * + * @see ContainerManagementProtocol#stopContainer(StopContainerRequest) + */ +@Public +@Stable +public abstract class StopContainersResponse { + @Private + @Unstable + public static StopContainersResponse newInstance( + List succeededRequests, + Map failedRequests) { + StopContainersResponse response = + Records.newRecord(StopContainersResponse.class); + response.setFailedRequests(failedRequests); + response.setSuccessfullyStoppedContainers(succeededRequests); + return response; + } + + /** + * Get the list of containerIds of successfully stopped containers. + * + * @return the list of containerIds of successfully stopped containers. + */ + @Public + @Stable + public abstract List getSuccessfullyStoppedContainers(); + + /** + * Set the list of containerIds of successfully stopped containers. + */ + @Private + @Unstable + public abstract void setSuccessfullyStoppedContainers( + List succeededRequests); + + /** + * Get the containerId-to-exception map in which the exception indicates error + * from per container for failed requests + */ + @Public + @Stable + public abstract Map getFailedRequests(); + + /** + * Set the containerId-to-exception map in which the exception indicates error + * from per container for failed requests + */ + @Private + @Unstable + public abstract void setFailedRequests( + Map failedRequests); +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java index 54c1d07..03f82ac 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java @@ -132,7 +132,7 @@ public static ContainerLaunchContext newInstance( /** *

- * Get application-specific binary service data. This is a map keyed + * Set application-specific binary service data. This is a map keyed * by the name of each {@link AuxiliaryService} that is configured on a * NodeManager and value correspond to the application specific data targeted * for the keyed {@link AuxiliaryService}. All pre-existing Map entries are diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SerializedException.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SerializedException.java new file mode 100644 index 0000000..aba54d4 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SerializedException.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.util.Records; + +@Private +@Unstable +public abstract class SerializedException { + + @Private + @Unstable + public static SerializedException newInstance(Throwable e) { + SerializedException exception = + Records.newRecord(SerializedException.class); + exception.init(e); + return exception; + } + + /** + * Constructs a new SerializedException with the specified detail + * message and cause. + */ + @Private + @Unstable + public abstract void init(String message, Throwable cause); + + /** + * Constructs a new SerializedException with the specified detail + * message. + */ + @Private + @Unstable + public abstract void init(String message); + + /** + * Constructs a new SerializedException with the specified cause. + */ + @Private + @Unstable + public abstract void init(Throwable cause); + + /** + * Get the detail message string of this exception. + * @return the detail message string of this exception. + */ + @Private + @Unstable + public abstract String getMessage(); + + /** + * Get the backtrace of this exception. + * @return the backtrace of this exception. + */ + @Private + @Unstable + public abstract String getRemoteTrace(); + + /** + * Get the cause of this exception or null if the cause is nonexistent or + * unknown. + * @return the cause of this exception. + */ + @Private + @Unstable + public abstract SerializedException getCause(); + + /** + * Deserialize the exception to a new Throwable. + * @return the Throwable form of this serialized exception. + */ + @Private + @Unstable + public abstract Throwable deSerialize(); +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java index e50098f..d28009c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java @@ -25,7 +25,6 @@ import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; import org.apache.hadoop.yarn.conf.YarnConfiguration; /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto index 5f02a28..98f438a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/containermanagement_protocol.proto @@ -30,7 +30,7 @@ option java_generate_equals_and_hash = true; import "yarn_service_protos.proto"; service ContainerManagementProtocolService { - rpc startContainer(StartContainerRequestProto) returns (StartContainerResponseProto); - rpc stopContainer(StopContainerRequestProto) returns (StopContainerResponseProto); - rpc getContainerStatus(GetContainerStatusRequestProto) returns (GetContainerStatusResponseProto); + rpc startContainers(StartContainersRequestProto) returns (StartContainersResponseProto); + rpc stopContainers(StopContainersRequestProto) returns (StopContainersResponseProto); + rpc getContainerStatuses(GetContainerStatusesRequestProto) returns (GetContainerStatusesResponseProto); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 4a6db02..bd009e0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -182,3 +182,37 @@ message GetContainerStatusRequestProto { message GetContainerStatusResponseProto { optional ContainerStatusProto status = 1; } + +//// bulk API records +message StartContainersRequestProto { + repeated StartContainerRequestProto start_container_request = 1; +} + +message ContainerExceptionMapProto { + optional ContainerIdProto container_id = 1; + optional SerializedExceptionProto exception = 2; +} + +message StartContainersResponseProto { + repeated StringBytesMapProto services_meta_data = 1; + repeated ContainerIdProto succeeded_requests = 2; + repeated ContainerExceptionMapProto failed_requests = 3; +} + +message StopContainersRequestProto { + repeated ContainerIdProto container_id = 1; +} + +message StopContainersResponseProto { + repeated ContainerIdProto succeeded_requests = 1; + repeated ContainerExceptionMapProto failed_requests = 2; +} + +message GetContainerStatusesRequestProto { + repeated ContainerIdProto container_id = 1; +} + +message GetContainerStatusesResponseProto { + repeated ContainerStatusProto status = 1; + repeated ContainerExceptionMapProto failed_requests = 2; +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java index 54a73fa..94610ff 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -30,9 +32,13 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; @@ -181,11 +187,22 @@ private void addStartingContainer(StartedContainer startedContainer) proxy = cmProxy.getProxy(container.getNodeId().toString(), container.getId()); - allServiceResponse = + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + container.getContainerToken()); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + StartContainersResponse response = proxy - .getContainerManagementProtocol().startContainer( - StartContainerRequest.newInstance(containerLaunchContext, - container.getContainerToken())).getAllServicesMetaData(); + .getContainerManagementProtocol().startContainers(allRequests); + if (response.getFailedRequests() != null + && response.getFailedRequests().containsKey(container.getId())) { + throw (YarnException) response.getFailedRequests() + .get(container.getId()).deSerialize(); + } + allServiceResponse = response.getAllServicesMetaData(); startingContainer.state = ContainerState.RUNNING; } catch (YarnException e) { startingContainer.state = ContainerState.COMPLETE; @@ -238,11 +255,19 @@ public ContainerStatus getContainerStatus(ContainerId containerId, NodeId nodeId) throws YarnException, IOException { ContainerManagementProtocolProxyData proxy = null; + List containerIds = new ArrayList(); + containerIds.add(containerId); try { proxy = cmProxy.getProxy(nodeId.toString(), containerId); - ContainerStatus containerStatus = - proxy.getContainerManagementProtocol().getContainerStatus( - GetContainerStatusRequest.newInstance(containerId)).getStatus(); + GetContainerStatusesResponse response = + proxy.getContainerManagementProtocol().getContainerStatuses( + GetContainerStatusesRequest.newInstance(containerIds)); + if (response.getFailedRequests() != null + && response.getFailedRequests().containsKey(containerId)) { + throw (YarnException) response.getFailedRequests().get(containerId) + .deSerialize(); + } + ContainerStatus containerStatus = response.getContainerStatuses().get(0); return containerStatus; } finally { if (proxy != null) { @@ -254,10 +279,18 @@ public ContainerStatus getContainerStatus(ContainerId containerId, private void stopContainerInternal(ContainerId containerId, NodeId nodeId) throws IOException, YarnException { ContainerManagementProtocolProxyData proxy = null; + List containerIds = new ArrayList(); + containerIds.add(containerId); try { proxy = cmProxy.getProxy(nodeId.toString(), containerId); - proxy.getContainerManagementProtocol().stopContainer( - StopContainerRequest.newInstance(containerId)); + StopContainersResponse response = + proxy.getContainerManagementProtocol().stopContainers( + StopContainersRequest.newInstance(containerIds)); + if (response.getFailedRequests() != null + && response.getFailedRequests().containsKey(containerId)) { + throw (YarnException) response.getFailedRequests().get(containerId) + .deSerialize(); + } } finally { if (proxy != null) { cmProxy.mayBeCloseProxy(proxy); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java index f6c0fa6..15397e3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java @@ -30,24 +30,24 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainerRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainerResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersResponsePBImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.RPCUtil; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainerRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersRequestProto; import com.google.protobuf.ServiceException; @@ -87,14 +87,14 @@ public void close() { } @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws YarnException, - IOException { - GetContainerStatusRequestProto requestProto = - ((GetContainerStatusRequestPBImpl) request).getProto(); + public StartContainersResponse + startContainers(StartContainersRequest requests) throws YarnException, + IOException { + StartContainersRequestProto requestProto = + ((StartContainersRequestPBImpl) requests).getProto(); try { - return new GetContainerStatusResponsePBImpl(proxy.getContainerStatus( - null, requestProto)); + return new StartContainersResponsePBImpl(proxy.startContainers(null, + requestProto)); } catch (ServiceException e) { RPCUtil.unwrapAndThrowException(e); return null; @@ -102,12 +102,12 @@ public GetContainerStatusResponse getContainerStatus( } @Override - public StartContainerResponse startContainer(StartContainerRequest request) + public StopContainersResponse stopContainers(StopContainersRequest requests) throws YarnException, IOException { - StartContainerRequestProto requestProto = - ((StartContainerRequestPBImpl) request).getProto(); + StopContainersRequestProto requestProto = + ((StopContainersRequestPBImpl) requests).getProto(); try { - return new StartContainerResponsePBImpl(proxy.startContainer(null, + return new StopContainersResponsePBImpl(proxy.stopContainers(null, requestProto)); } catch (ServiceException e) { RPCUtil.unwrapAndThrowException(e); @@ -116,13 +116,13 @@ public StartContainerResponse startContainer(StartContainerRequest request) } @Override - public StopContainerResponse stopContainer(StopContainerRequest request) - throws YarnException, IOException { - StopContainerRequestProto requestProto = - ((StopContainerRequestPBImpl) request).getProto(); + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException, IOException { + GetContainerStatusesRequestProto requestProto = + ((GetContainerStatusesRequestPBImpl) request).getProto(); try { - return new StopContainerResponsePBImpl(proxy.stopContainer(null, - requestProto)); + return new GetContainerStatusesResponsePBImpl(proxy.getContainerStatuses( + null, requestProto)); } catch (ServiceException e) { RPCUtil.unwrapAndThrowException(e); return null; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java index 5c2a96a..2d33e69 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java @@ -23,22 +23,22 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerResponsePBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainerRequestPBImpl; -import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainerResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersResponsePBImpl; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainerRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainerResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersResponseProto; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -53,12 +53,12 @@ public ContainerManagementProtocolPBServiceImpl(ContainerManagementProtocol impl } @Override - public GetContainerStatusResponseProto getContainerStatus(RpcController arg0, - GetContainerStatusRequestProto proto) throws ServiceException { - GetContainerStatusRequestPBImpl request = new GetContainerStatusRequestPBImpl(proto); + public StartContainersResponseProto startContainers(RpcController arg0, + StartContainersRequestProto proto) throws ServiceException { + StartContainersRequestPBImpl request = new StartContainersRequestPBImpl(proto); try { - GetContainerStatusResponse response = real.getContainerStatus(request); - return ((GetContainerStatusResponsePBImpl)response).getProto(); + StartContainersResponse response = real.startContainers(request); + return ((StartContainersResponsePBImpl)response).getProto(); } catch (YarnException e) { throw new ServiceException(e); } catch (IOException e) { @@ -67,12 +67,12 @@ public GetContainerStatusResponseProto getContainerStatus(RpcController arg0, } @Override - public StartContainerResponseProto startContainer(RpcController arg0, - StartContainerRequestProto proto) throws ServiceException { - StartContainerRequestPBImpl request = new StartContainerRequestPBImpl(proto); + public StopContainersResponseProto stopContainers(RpcController arg0, + StopContainersRequestProto proto) throws ServiceException { + StopContainersRequestPBImpl request = new StopContainersRequestPBImpl(proto); try { - StartContainerResponse response = real.startContainer(request); - return ((StartContainerResponsePBImpl)response).getProto(); + StopContainersResponse response = real.stopContainers(request); + return ((StopContainersResponsePBImpl)response).getProto(); } catch (YarnException e) { throw new ServiceException(e); } catch (IOException e) { @@ -81,17 +81,17 @@ public StartContainerResponseProto startContainer(RpcController arg0, } @Override - public StopContainerResponseProto stopContainer(RpcController arg0, - StopContainerRequestProto proto) throws ServiceException { - StopContainerRequestPBImpl request = new StopContainerRequestPBImpl(proto); + public GetContainerStatusesResponseProto getContainerStatuses( + RpcController arg0, GetContainerStatusesRequestProto proto) + throws ServiceException { + GetContainerStatusesRequestPBImpl request = new GetContainerStatusesRequestPBImpl(proto); try { - StopContainerResponse response = real.stopContainer(request); - return ((StopContainerResponsePBImpl)response).getProto(); + GetContainerStatusesResponse response = real.getContainerStatuses(request); + return ((GetContainerStatusesResponsePBImpl)response).getProto(); } catch (YarnException e) { throw new ServiceException(e); } catch (IOException e) { throw new ServiceException(e); } } - } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusRequestPBImpl.java deleted file mode 100644 index fa88d1f..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusRequestPBImpl.java +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; - - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusRequestProtoOrBuilder; - -@Private -@Unstable -public class GetContainerStatusRequestPBImpl extends GetContainerStatusRequest { - GetContainerStatusRequestProto proto = GetContainerStatusRequestProto.getDefaultInstance(); - GetContainerStatusRequestProto.Builder builder = null; - boolean viaProto = false; - - private ContainerId containerId = null; - - - public GetContainerStatusRequestPBImpl() { - builder = GetContainerStatusRequestProto.newBuilder(); - } - - public GetContainerStatusRequestPBImpl(GetContainerStatusRequestProto proto) { - this.proto = proto; - viaProto = true; - } - - public GetContainerStatusRequestProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); - } - - private void mergeLocalToBuilder() { - if (this.containerId != null) { - builder.setContainerId(convertToProtoFormat(this.containerId)); - } - } - - private void mergeLocalToProto() { - if (viaProto) - maybeInitBuilder(); - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = GetContainerStatusRequestProto.newBuilder(proto); - } - viaProto = false; - } - - - @Override - public ContainerId getContainerId() { - GetContainerStatusRequestProtoOrBuilder p = viaProto ? proto : builder; - if (this.containerId != null) { - return this.containerId; - } - if (!p.hasContainerId()) { - return null; - } - this.containerId = convertFromProtoFormat(p.getContainerId()); - return this.containerId; - } - - @Override - public void setContainerId(ContainerId containerId) { - maybeInitBuilder(); - if (containerId == null) - builder.clearContainerId(); - this.containerId = containerId; - } - - private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { - return new ContainerIdPBImpl(p); - } - - private ContainerIdProto convertToProtoFormat(ContainerId t) { - return ((ContainerIdPBImpl)t).getProto(); - } - - - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusResponsePBImpl.java deleted file mode 100644 index 2b635f9..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusResponsePBImpl.java +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; - - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusResponseProtoOrBuilder; - -@Private -@Unstable -public class GetContainerStatusResponsePBImpl extends GetContainerStatusResponse { - GetContainerStatusResponseProto proto = GetContainerStatusResponseProto.getDefaultInstance(); - GetContainerStatusResponseProto.Builder builder = null; - boolean viaProto = false; - - private ContainerStatus containerStatus = null; - - - public GetContainerStatusResponsePBImpl() { - builder = GetContainerStatusResponseProto.newBuilder(); - } - - public GetContainerStatusResponsePBImpl(GetContainerStatusResponseProto proto) { - this.proto = proto; - viaProto = true; - } - - public GetContainerStatusResponseProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); - } - - private void mergeLocalToBuilder() { - if (this.containerStatus != null) { - builder.setStatus(convertToProtoFormat(this.containerStatus)); - } - } - - private void mergeLocalToProto() { - if (viaProto) - maybeInitBuilder(); - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = GetContainerStatusResponseProto.newBuilder(proto); - } - viaProto = false; - } - - - @Override - public ContainerStatus getStatus() { - GetContainerStatusResponseProtoOrBuilder p = viaProto ? proto : builder; - if (this.containerStatus != null) { - return this.containerStatus; - } - if (!p.hasStatus()) { - return null; - } - this.containerStatus = convertFromProtoFormat(p.getStatus()); - return this.containerStatus; - } - - @Override - public void setStatus(ContainerStatus status) { - maybeInitBuilder(); - if (status == null) - builder.clearStatus(); - this.containerStatus = status; - } - - private ContainerStatusPBImpl convertFromProtoFormat(ContainerStatusProto p) { - return new ContainerStatusPBImpl(p); - } - - private ContainerStatusProto convertToProtoFormat(ContainerStatus t) { - return ((ContainerStatusPBImpl)t).getProto(); - } - - - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusesRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusesRequestPBImpl.java new file mode 100644 index 0000000..0c305ca --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusesRequestPBImpl.java @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesRequestProtoOrBuilder; + +@Private +@Unstable +public class GetContainerStatusesRequestPBImpl extends + GetContainerStatusesRequest { + GetContainerStatusesRequestProto proto = GetContainerStatusesRequestProto + .getDefaultInstance(); + GetContainerStatusesRequestProto.Builder builder = null; + boolean viaProto = false; + + private List containerIds = null; + + public GetContainerStatusesRequestPBImpl() { + builder = GetContainerStatusesRequestProto.newBuilder(); + } + + public GetContainerStatusesRequestPBImpl( + GetContainerStatusesRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetContainerStatusesRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return getProto().toString().replaceAll("\\n", ", ") + .replaceAll("\\s+", " "); + } + + private void mergeLocalToBuilder() { + if (this.containerIds != null) { + addLocalContainerIdsToProto(); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetContainerStatusesRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void addLocalContainerIdsToProto() { + maybeInitBuilder(); + builder.clearContainerId(); + if (this.containerIds == null) + return; + List protoList = new ArrayList(); + for (ContainerId id : containerIds) { + protoList.add(convertToProtoFormat(id)); + } + builder.addAllContainerId(protoList); + } + + private void initLocalContainerIds() { + if (this.containerIds != null) { + return; + } + GetContainerStatusesRequestProtoOrBuilder p = viaProto ? proto : builder; + List containerIds = p.getContainerIdList(); + this.containerIds = new ArrayList(); + for (ContainerIdProto id : containerIds) { + this.containerIds.add(convertFromProtoFormat(id)); + } + } + + @Override + public List getContainerIds() { + initLocalContainerIds(); + return this.containerIds; + } + + @Override + public void setContainerIds(List containerIds) { + maybeInitBuilder(); + if (containerIds == null) + builder.clearContainerId(); + this.containerIds = containerIds; + } + + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { + return new ContainerIdPBImpl(p); + } + + private ContainerIdProto convertToProtoFormat(ContainerId t) { + return ((ContainerIdPBImpl) t).getProto(); + } + +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusesResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusesResponsePBImpl.java new file mode 100644 index 0000000..18df214 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetContainerStatusesResponsePBImpl.java @@ -0,0 +1,223 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.SerializedException; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto; +import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ContainerExceptionMapProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesResponseProtoOrBuilder; + +@Private +@Unstable +public class GetContainerStatusesResponsePBImpl extends + GetContainerStatusesResponse { + GetContainerStatusesResponseProto proto = GetContainerStatusesResponseProto + .getDefaultInstance(); + GetContainerStatusesResponseProto.Builder builder = null; + boolean viaProto = false; + + private List containerStatuses = null; + private Map failedRequests = null; + + public GetContainerStatusesResponsePBImpl() { + builder = GetContainerStatusesResponseProto.newBuilder(); + } + + public GetContainerStatusesResponsePBImpl( + GetContainerStatusesResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetContainerStatusesResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return getProto().toString().replaceAll("\\n", ", ") + .replaceAll("\\s+", " "); + } + + private void mergeLocalToBuilder() { + if (this.containerStatuses != null) { + addLocalContainerStatusesToProto(); + } + if (this.failedRequests != null) { + addFailedRequestsToProto(); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetContainerStatusesResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void addLocalContainerStatusesToProto() { + maybeInitBuilder(); + builder.clearStatus(); + if (this.containerStatuses == null) + return; + List protoList = + new ArrayList(); + for (ContainerStatus status : containerStatuses) { + protoList.add(convertToProtoFormat(status)); + } + builder.addAllStatus(protoList); + } + + private void addFailedRequestsToProto() { + maybeInitBuilder(); + builder.clearFailedRequests(); + if (this.failedRequests == null) + return; + List protoList = + new ArrayList(); + for (Map.Entry entry : this.failedRequests + .entrySet()) { + protoList.add(ContainerExceptionMapProto.newBuilder() + .setContainerId(convertToProtoFormat(entry.getKey())) + .setException(convertToProtoFormat(entry.getValue())).build()); + } + builder.addAllFailedRequests(protoList); + } + + private void initLocalContainerStatuses() { + if (this.containerStatuses != null) { + return; + } + GetContainerStatusesResponseProtoOrBuilder p = viaProto ? proto : builder; + List statuses = p.getStatusList(); + this.containerStatuses = new ArrayList(); + for (ContainerStatusProto status : statuses) { + this.containerStatuses.add(convertFromProtoFormat(status)); + } + } + + private void initFailedRequests() { + if (this.failedRequests != null) { + return; + } + GetContainerStatusesResponseProtoOrBuilder p = viaProto ? proto : builder; + List protoList = p.getFailedRequestsList(); + this.failedRequests = new HashMap(); + for (ContainerExceptionMapProto ce : protoList) { + this.failedRequests.put(convertFromProtoFormat(ce.getContainerId()), + convertFromProtoFormat(ce.getException())); + } + } + + @Override + public List getContainerStatuses() { + initLocalContainerStatuses(); + return this.containerStatuses; + } + + @Override + public void setContainerStatuses(List statuses) { + maybeInitBuilder(); + if (statuses == null) + builder.clearStatus(); + this.containerStatuses = statuses; + } + + @Override + public Map getFailedRequests() { + initFailedRequests(); + return this.failedRequests; + } + + @Override + public void setFailedRequests( + Map failedRequests) { + maybeInitBuilder(); + if (failedRequests == null) + builder.clearFailedRequests(); + this.failedRequests = failedRequests; + } + + private ContainerStatusPBImpl convertFromProtoFormat(ContainerStatusProto p) { + return new ContainerStatusPBImpl(p); + } + + private ContainerStatusProto convertToProtoFormat(ContainerStatus t) { + return ((ContainerStatusPBImpl) t).getProto(); + } + + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { + return new ContainerIdPBImpl(p); + } + + private ContainerIdProto convertToProtoFormat(ContainerId t) { + return ((ContainerIdPBImpl) t).getProto(); + } + + private SerializedExceptionPBImpl convertFromProtoFormat( + SerializedExceptionProto p) { + return new SerializedExceptionPBImpl(p); + } + + private SerializedExceptionProto convertToProtoFormat(SerializedException t) { + return ((SerializedExceptionPBImpl) t).getProto(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java deleted file mode 100644 index 8edbfa7..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java +++ /dev/null @@ -1,176 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; - - -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; -import org.apache.hadoop.yarn.proto.YarnProtos.StringBytesMapProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerResponseProtoOrBuilder; - -import com.google.protobuf.ByteString; - -@Private -@Unstable -public class StartContainerResponsePBImpl extends StartContainerResponse { - StartContainerResponseProto proto = StartContainerResponseProto.getDefaultInstance(); - StartContainerResponseProto.Builder builder = null; - boolean viaProto = false; - - private Map servicesMetaData = null; - - public StartContainerResponsePBImpl() { - builder = StartContainerResponseProto.newBuilder(); - } - - public StartContainerResponsePBImpl(StartContainerResponseProto proto) { - this.proto = proto; - viaProto = true; - } - - public synchronized StartContainerResponseProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); - } - - private synchronized void mergeLocalToBuilder() { - if (this.servicesMetaData != null) { - addServicesMetaDataToProto(); - } - } - - protected final ByteBuffer convertFromProtoFormat(ByteString byteString) { - return ProtoUtils.convertFromProtoFormat(byteString); - } - - protected final ByteString convertToProtoFormat(ByteBuffer byteBuffer) { - return ProtoUtils.convertToProtoFormat(byteBuffer); - } - - private synchronized void mergeLocalToProto() { - if (viaProto) { - maybeInitBuilder(); - } - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private synchronized void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = StartContainerResponseProto.newBuilder(proto); - } - viaProto = false; - } - - - @Override - public synchronized Map getAllServicesMetaData() { - initServicesMetaData(); - return this.servicesMetaData; - } - @Override - public synchronized void setAllServicesMetaData( - Map servicesMetaData) { - if(servicesMetaData == null) { - return; - } - initServicesMetaData(); - this.servicesMetaData.clear(); - this.servicesMetaData.putAll(servicesMetaData); - } - - private synchronized void initServicesMetaData() { - if (this.servicesMetaData != null) { - return; - } - StartContainerResponseProtoOrBuilder p = viaProto ? proto : builder; - List list = p.getServicesMetaDataList(); - this.servicesMetaData = new HashMap(); - - for (StringBytesMapProto c : list) { - this.servicesMetaData.put(c.getKey(), convertFromProtoFormat(c.getValue())); - } - } - - private synchronized void addServicesMetaDataToProto() { - maybeInitBuilder(); - builder.clearServicesMetaData(); - if (servicesMetaData == null) - return; - Iterable iterable = new Iterable() { - - @Override - public synchronized Iterator iterator() { - return new Iterator() { - - Iterator keyIter = servicesMetaData.keySet().iterator(); - - @Override - public synchronized void remove() { - throw new UnsupportedOperationException(); - } - - @Override - public synchronized StringBytesMapProto next() { - String key = keyIter.next(); - return StringBytesMapProto.newBuilder().setKey(key).setValue(convertToProtoFormat(servicesMetaData.get(key))).build(); - } - - @Override - public synchronized boolean hasNext() { - return keyIter.hasNext(); - } - }; - } - }; - builder.addAllServicesMetaData(iterable); - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainersRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainersRequestPBImpl.java new file mode 100644 index 0000000..2233705 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainersRequestPBImpl.java @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersRequestProtoOrBuilder; + +public class StartContainersRequestPBImpl extends StartContainersRequest { + StartContainersRequestProto proto = StartContainersRequestProto + .getDefaultInstance(); + StartContainersRequestProto.Builder builder = null; + boolean viaProto = false; + + private List requests = null; + + public StartContainersRequestPBImpl() { + builder = StartContainersRequestProto.newBuilder(); + } + + public StartContainersRequestPBImpl(StartContainersRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public StartContainersRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (requests != null) { + addLocalRequestsToProto(); + } + } + + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = StartContainersRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void addLocalRequestsToProto() { + maybeInitBuilder(); + builder.clearStartContainerRequest(); + List protoList = + new ArrayList(); + for (StartContainerRequest r : this.requests) { + protoList.add(convertToProtoFormat(r)); + } + builder.addAllStartContainerRequest(protoList); + } + + private void initLocalRequests() { + StartContainersRequestProtoOrBuilder p = viaProto ? proto : builder; + List requestList = + p.getStartContainerRequestList(); + this.requests = new ArrayList(); + for (StartContainerRequestProto r : requestList) { + this.requests.add(convertFromProtoFormat(r)); + } + } + + @Override + public void setStartContainerRequests(List requests) { + maybeInitBuilder(); + if (requests == null) { + builder.clearStartContainerRequest(); + } + this.requests = requests; + } + + @Override + public List getStartContainerRequests() { + if (this.requests != null) { + return this.requests; + } + initLocalRequests(); + return this.requests; + } + + private StartContainerRequestPBImpl convertFromProtoFormat( + StartContainerRequestProto p) { + return new StartContainerRequestPBImpl(p); + } + + private StartContainerRequestProto convertToProtoFormat( + StartContainerRequest t) { + return ((StartContainerRequestPBImpl) t).getProto(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainersResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainersResponsePBImpl.java new file mode 100644 index 0000000..1482cd7 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainersResponsePBImpl.java @@ -0,0 +1,319 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.SerializedException; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; +import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto; +import org.apache.hadoop.yarn.proto.YarnProtos.StringBytesMapProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ContainerExceptionMapProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersResponseProtoOrBuilder; + +import com.google.protobuf.ByteString; + +@Private +@Unstable +public class StartContainersResponsePBImpl extends StartContainersResponse { + StartContainersResponseProto proto = StartContainersResponseProto + .getDefaultInstance(); + StartContainersResponseProto.Builder builder = null; + boolean viaProto = false; + + private Map servicesMetaData = null; + private List succeededContainers = null; + private Map failedContainers = null; + + public StartContainersResponsePBImpl() { + builder = StartContainersResponseProto.newBuilder(); + } + + public StartContainersResponsePBImpl(StartContainersResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public StartContainersResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return getProto().toString().replaceAll("\\n", ", ") + .replaceAll("\\s+", " "); + } + + private void mergeLocalToBuilder() { + if (this.servicesMetaData != null) { + addServicesMetaDataToProto(); + } + if (this.succeededContainers != null) { + addSucceededContainersToProto(); + } + if (this.failedContainers != null) { + addFailedContainersToProto(); + } + } + + protected final ByteBuffer convertFromProtoFormat(ByteString byteString) { + return ProtoUtils.convertFromProtoFormat(byteString); + } + + protected final ByteString convertToProtoFormat(ByteBuffer byteBuffer) { + return ProtoUtils.convertToProtoFormat(byteBuffer); + } + + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { + return new ContainerIdPBImpl(p); + } + + private ContainerIdProto convertToProtoFormat(ContainerId t) { + return ((ContainerIdPBImpl) t).getProto(); + } + + private SerializedExceptionPBImpl convertFromProtoFormat( + SerializedExceptionProto p) { + return new SerializedExceptionPBImpl(p); + } + + private SerializedExceptionProto convertToProtoFormat(SerializedException t) { + return ((SerializedExceptionPBImpl) t).getProto(); + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = StartContainersResponseProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public Map getAllServicesMetaData() { + initServicesMetaData(); + return this.servicesMetaData; + } + + @Override + public void setAllServicesMetaData(Map servicesMetaData) { + if (servicesMetaData == null) { + return; + } + initServicesMetaData(); + this.servicesMetaData.clear(); + this.servicesMetaData.putAll(servicesMetaData); + } + + private void initServicesMetaData() { + if (this.servicesMetaData != null) { + return; + } + StartContainersResponseProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getServicesMetaDataList(); + this.servicesMetaData = new HashMap(); + + for (StringBytesMapProto c : list) { + this.servicesMetaData.put(c.getKey(), + convertFromProtoFormat(c.getValue())); + } + } + + private void addServicesMetaDataToProto() { + maybeInitBuilder(); + builder.clearServicesMetaData(); + if (servicesMetaData == null) + return; + Iterable iterable = + new Iterable() { + + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator keyIter = servicesMetaData.keySet().iterator(); + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public StringBytesMapProto next() { + String key = keyIter.next(); + return StringBytesMapProto.newBuilder().setKey(key) + .setValue(convertToProtoFormat(servicesMetaData.get(key))) + .build(); + } + + @Override + public boolean hasNext() { + return keyIter.hasNext(); + } + }; + } + }; + builder.addAllServicesMetaData(iterable); + } + + private void addFailedContainersToProto() { + maybeInitBuilder(); + builder.clearFailedRequests(); + if (this.failedContainers == null) + return; + List protoList = + new ArrayList(); + + for (Map.Entry entry : this.failedContainers + .entrySet()) { + protoList.add(ContainerExceptionMapProto.newBuilder() + .setContainerId(convertToProtoFormat(entry.getKey())) + .setException(convertToProtoFormat(entry.getValue())).build()); + } + builder.addAllFailedRequests(protoList); + } + + private void addSucceededContainersToProto() { + maybeInitBuilder(); + builder.clearSucceededRequests(); + if (this.succeededContainers == null) { + return; + } + Iterable iterable = new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator iter = succeededContainers.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public ContainerIdProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + + } + }; + } + }; + builder.addAllSucceededRequests(iterable); + } + + private void initSucceededContainers() { + if (this.succeededContainers != null) + return; + StartContainersResponseProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getSucceededRequestsList(); + this.succeededContainers = new ArrayList(); + for (ContainerIdProto c : list) { + this.succeededContainers.add(convertFromProtoFormat(c)); + } + } + + @Override + public List getSuccessfullyStartedContainers() { + initSucceededContainers(); + return this.succeededContainers; + } + + @Override + public void setSuccessfullyStartedContainers( + List succeededContainers) { + maybeInitBuilder(); + if (succeededContainers == null) { + builder.clearSucceededRequests(); + } + this.succeededContainers = succeededContainers; + } + + private void initFailedContainers() { + if (this.failedContainers != null) { + return; + } + StartContainersResponseProtoOrBuilder p = viaProto ? proto : builder; + List protoList = p.getFailedRequestsList(); + this.failedContainers = new HashMap(); + for (ContainerExceptionMapProto ce : protoList) { + this.failedContainers.put(convertFromProtoFormat(ce.getContainerId()), + convertFromProtoFormat(ce.getException())); + } + } + + @Override + public Map getFailedRequests() { + initFailedContainers(); + return this.failedContainers; + } + + @Override + public void setFailedRequests( + Map failedContainers) { + maybeInitBuilder(); + if (failedContainers == null) + builder.clearFailedRequests(); + this.failedContainers = failedContainers; + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainerRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainerRequestPBImpl.java deleted file mode 100644 index dfba527..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainerRequestPBImpl.java +++ /dev/null @@ -1,130 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; - - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainerRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainerRequestProtoOrBuilder; - -@Private -@Unstable -public class StopContainerRequestPBImpl extends StopContainerRequest { - StopContainerRequestProto proto = StopContainerRequestProto.getDefaultInstance(); - StopContainerRequestProto.Builder builder = null; - boolean viaProto = false; - - private ContainerId containerId = null; - - - public StopContainerRequestPBImpl() { - builder = StopContainerRequestProto.newBuilder(); - } - - public StopContainerRequestPBImpl(StopContainerRequestProto proto) { - this.proto = proto; - viaProto = true; - } - - public StopContainerRequestProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); - } - - private void mergeLocalToBuilder() { - if (this.containerId != null) { - builder.setContainerId(convertToProtoFormat(this.containerId)); - } - } - - private void mergeLocalToProto() { - if (viaProto) - maybeInitBuilder(); - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = StopContainerRequestProto.newBuilder(proto); - } - viaProto = false; - } - - - @Override - public ContainerId getContainerId() { - StopContainerRequestProtoOrBuilder p = viaProto ? proto : builder; - if (this.containerId != null) { - return this.containerId; - } - if (!p.hasContainerId()) { - return null; - } - this.containerId = convertFromProtoFormat(p.getContainerId()); - return this.containerId; - } - - @Override - public void setContainerId(ContainerId containerId) { - maybeInitBuilder(); - if (containerId == null) - builder.clearContainerId(); - this.containerId = containerId; - } - - private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { - return new ContainerIdPBImpl(p); - } - - private ContainerIdProto convertToProtoFormat(ContainerId t) { - return ((ContainerIdPBImpl)t).getProto(); - } - - - -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainerResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainerResponsePBImpl.java deleted file mode 100644 index 6e3fe21..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainerResponsePBImpl.java +++ /dev/null @@ -1,68 +0,0 @@ -/** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ - -package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; - - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainerResponseProto; - -@Private -@Unstable -public class StopContainerResponsePBImpl extends StopContainerResponse { - StopContainerResponseProto proto = StopContainerResponseProto.getDefaultInstance(); - StopContainerResponseProto.Builder builder = null; - boolean viaProto = false; - - public StopContainerResponsePBImpl() { - builder = StopContainerResponseProto.newBuilder(); - } - - public StopContainerResponsePBImpl(StopContainerResponseProto proto) { - this.proto = proto; - viaProto = true; - } - - public StopContainerResponseProto getProto() { - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " "); - } -} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainersRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainersRequestPBImpl.java new file mode 100644 index 0000000..5c758e8 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainersRequestPBImpl.java @@ -0,0 +1,146 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersRequestProtoOrBuilder; + +@Private +@Unstable +public class StopContainersRequestPBImpl extends StopContainersRequest { + StopContainersRequestProto proto = StopContainersRequestProto + .getDefaultInstance(); + StopContainersRequestProto.Builder builder = null; + boolean viaProto = false; + + private List containerIds = null; + + public StopContainersRequestPBImpl() { + builder = StopContainersRequestProto.newBuilder(); + } + + public StopContainersRequestPBImpl(StopContainersRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public StopContainersRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return getProto().toString().replaceAll("\\n", ", ") + .replaceAll("\\s+", " "); + } + + private void mergeLocalToBuilder() { + if (this.containerIds != null) { + addLocalContainerIdsToProto(); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = StopContainersRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void addLocalContainerIdsToProto() { + maybeInitBuilder(); + builder.clearContainerId(); + if (this.containerIds == null) + return; + List protoList = new ArrayList(); + for (ContainerId id : containerIds) { + protoList.add(convertToProtoFormat(id)); + } + builder.addAllContainerId(protoList); + } + + private void initLocalContainerIds() { + if (this.containerIds != null) { + return; + } + StopContainersRequestProtoOrBuilder p = viaProto ? proto : builder; + List containerIds = p.getContainerIdList(); + this.containerIds = new ArrayList(); + for (ContainerIdProto id : containerIds) { + this.containerIds.add(convertFromProtoFormat(id)); + } + } + + @Override + public List getContainerIds() { + initLocalContainerIds(); + return this.containerIds; + } + + @Override + public void setContainerIds(List containerIds) { + maybeInitBuilder(); + if (containerIds == null) + builder.clearContainerId(); + this.containerIds = containerIds; + } + + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { + return new ContainerIdPBImpl(p); + } + + private ContainerIdProto convertToProtoFormat(ContainerId t) { + return ((ContainerIdPBImpl) t).getProto(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainersResponsePBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainersResponsePBImpl.java new file mode 100644 index 0000000..4d07dd8 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StopContainersResponsePBImpl.java @@ -0,0 +1,234 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.SerializedException; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.ContainerExceptionMapProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersResponseProtoOrBuilder; + +@Private +@Unstable +public class StopContainersResponsePBImpl extends StopContainersResponse { + StopContainersResponseProto proto = StopContainersResponseProto + .getDefaultInstance(); + StopContainersResponseProto.Builder builder = null; + boolean viaProto = false; + private List succeededRequests = null; + private Map failedRequests = null; + + public StopContainersResponsePBImpl() { + builder = StopContainersResponseProto.newBuilder(); + } + + public StopContainersResponsePBImpl(StopContainersResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public StopContainersResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return getProto().toString().replaceAll("\\n", ", ") + .replaceAll("\\s+", " "); + } + + private synchronized void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private synchronized void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = StopContainersResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private synchronized void mergeLocalToBuilder() { + + if (this.succeededRequests != null) { + addSucceededRequestsToProto(); + } + if (this.failedRequests != null) { + addFailedRequestsToProto(); + } + } + + private void addSucceededRequestsToProto() { + maybeInitBuilder(); + builder.clearSucceededRequests(); + if (this.succeededRequests == null) { + return; + } + Iterable iterable = new Iterable() { + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator iter = succeededRequests.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public ContainerIdProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + + } + }; + } + }; + builder.addAllSucceededRequests(iterable); + } + + private void addFailedRequestsToProto() { + maybeInitBuilder(); + builder.clearFailedRequests(); + if (this.failedRequests == null) + return; + List protoList = + new ArrayList(); + + for (Map.Entry entry : this.failedRequests + .entrySet()) { + protoList.add(ContainerExceptionMapProto.newBuilder() + .setContainerId(convertToProtoFormat(entry.getKey())) + .setException(convertToProtoFormat(entry.getValue())).build()); + } + builder.addAllFailedRequests(protoList); + } + + private void initSucceededRequests() { + if (this.succeededRequests != null) + return; + StopContainersResponseProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getSucceededRequestsList(); + this.succeededRequests = new ArrayList(); + for (ContainerIdProto c : list) { + this.succeededRequests.add(convertFromProtoFormat(c)); + } + } + + private void initFailedRequests() { + if (this.failedRequests != null) { + return; + } + StopContainersResponseProtoOrBuilder p = viaProto ? proto : builder; + List protoList = p.getFailedRequestsList(); + this.failedRequests = new HashMap(); + for (ContainerExceptionMapProto ce : protoList) { + this.failedRequests.put(convertFromProtoFormat(ce.getContainerId()), + convertFromProtoFormat(ce.getException())); + } + } + + @Override + public List getSuccessfullyStoppedContainers() { + initSucceededRequests(); + return this.succeededRequests; + } + + @Override + public void setSuccessfullyStoppedContainers(List succeededRequests) { + maybeInitBuilder(); + if (succeededRequests == null) { + builder.clearSucceededRequests(); + } + this.succeededRequests = succeededRequests; + } + + @Override + public Map getFailedRequests() { + initFailedRequests(); + return this.failedRequests; + } + + @Override + public void setFailedRequests( + Map failedRequests) { + maybeInitBuilder(); + if (failedRequests == null) + builder.clearFailedRequests(); + this.failedRequests = failedRequests; + } + + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { + return new ContainerIdPBImpl(p); + } + + private ContainerIdProto convertToProtoFormat(ContainerId t) { + return ((ContainerIdPBImpl) t).getProto(); + } + + private SerializedExceptionPBImpl convertFromProtoFormat( + SerializedExceptionProto p) { + return new SerializedExceptionPBImpl(p); + } + + private SerializedExceptionProto convertToProtoFormat(SerializedException t) { + return ((SerializedExceptionPBImpl) t).getProto(); + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java new file mode 100644 index 0000000..ae81d3a --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java @@ -0,0 +1,171 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.records.impl.pb; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; + +import org.apache.hadoop.yarn.api.records.SerializedException; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto; +import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProtoOrBuilder; + +public class SerializedExceptionPBImpl extends SerializedException { + + SerializedExceptionProto proto = SerializedExceptionProto + .getDefaultInstance(); + SerializedExceptionProto.Builder builder = null; + boolean viaProto = false; + + public SerializedExceptionPBImpl() { + } + + public SerializedExceptionPBImpl(SerializedExceptionProto proto) { + this.proto = proto; + viaProto = true; + } + + private SerializedExceptionPBImpl(Throwable t) { + init(t); + } + + public void init(String message) { + maybeInitBuilder(); + builder.setMessage(message); + } + + public void init(Throwable t) { + maybeInitBuilder(); + if (t == null) { + return; + } + + if (t.getCause() == null) { + } else { + builder.setCause(new SerializedExceptionPBImpl(t.getCause()).getProto()); + } + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + t.printStackTrace(pw); + pw.close(); + if (sw.toString() != null) + builder.setTrace(sw.toString()); + if (t.getMessage() != null) + builder.setMessage(t.getMessage()); + builder.setClassName(t.getClass().getCanonicalName()); + } + + public void init(String message, Throwable t) { + init(t); + if (message != null) + builder.setMessage(message); + } + + @Override + public Throwable deSerialize() { + + SerializedException cause = getCause(); + Throwable throwable = null; + SerializedExceptionProtoOrBuilder p = viaProto ? proto : builder; + Class realClass = null; + try { + realClass = Class.forName(p.getClassName()); + } catch (ClassNotFoundException e) { + throw new YarnRuntimeException(e); + } + if (YarnException.class.isAssignableFrom(realClass)) { + throwable = instantiateException(realClass.asSubclass(YarnException.class), + getMessage(), cause == null ? null: cause.deSerialize()); + } else if (IOException.class.isAssignableFrom(realClass)) { + throwable = instantiateException(realClass.asSubclass(IOException.class), + getMessage(), cause == null ? null : cause.deSerialize()); + } else if (RuntimeException.class.isAssignableFrom(realClass)) { + throwable = instantiateException(realClass.asSubclass(RuntimeException.class), + getMessage(), cause == null ? null : cause.deSerialize()); + } else { + throwable = instantiateException(realClass.asSubclass(Exception.class), + getMessage(), cause == null ? null : cause.deSerialize()); + } + return throwable; + } + + @Override + public String getMessage() { + SerializedExceptionProtoOrBuilder p = viaProto ? proto : builder; + return p.getMessage(); + } + + @Override + public String getRemoteTrace() { + SerializedExceptionProtoOrBuilder p = viaProto ? proto : builder; + return p.getTrace(); + } + + @Override + public SerializedException getCause() { + SerializedExceptionProtoOrBuilder p = viaProto ? proto : builder; + if (p.hasCause()) { + return new SerializedExceptionPBImpl(p.getCause()); + } else { + return null; + } + } + + public SerializedExceptionProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = SerializedExceptionProto.newBuilder(proto); + } + viaProto = false; + } + + private static T instantiateException( + Class cls, String message, Throwable cause) { + Constructor cn; + T ex = null; + try { + cn = cls.getConstructor(String.class); + cn.setAccessible(true); + ex = cn.newInstance(message); + ex.initCause(cause); + } catch (SecurityException e) { + throw new YarnRuntimeException(e); + } catch (NoSuchMethodException e) { + throw new YarnRuntimeException(e); + } catch (IllegalArgumentException e) { + throw new YarnRuntimeException(e); + } catch (InstantiationException e) { + throw new YarnRuntimeException(e); + } catch (IllegalAccessException e) { + throw new YarnRuntimeException(e); + } catch (InvocationTargetException e) { + throw new YarnRuntimeException(e); + } + return ex; + } +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java index 0c633f1..8fe5c3c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java @@ -21,6 +21,8 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.List; import junit.framework.Assert; @@ -31,12 +33,13 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -104,12 +107,15 @@ private void testRPCTimeout(String rpcClass) throws Exception { TestRPC.newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier); - StartContainerRequest scRequest = recordFactory - .newRecordInstance(StartContainerRequest.class); - scRequest.setContainerLaunchContext(containerLaunchContext); - scRequest.setContainerToken(containerToken); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + containerToken); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); try { - proxy.startContainer(scRequest); + proxy.startContainers(allRequests); } catch (Exception e) { LOG.info(StringUtils.stringifyException(e)); Assert.assertEquals("Error, exception is not: " @@ -129,17 +135,8 @@ private void testRPCTimeout(String rpcClass) throws Exception { private ContainerStatus status = null; @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws YarnException { - GetContainerStatusResponse response = recordFactory - .newRecordInstance(GetContainerStatusResponse.class); - response.setStatus(status); - return response; - } - - @Override - public StartContainerResponse startContainer(StartContainerRequest request) - throws YarnException, IOException { + public StartContainersResponse startContainers( + StartContainersRequest requests) throws YarnException, IOException { try { // make the thread sleep to look like its not going to respond Thread.sleep(10000); @@ -151,11 +148,22 @@ public StartContainerResponse startContainer(StartContainerRequest request) } @Override - public StopContainerResponse stopContainer(StopContainerRequest request) - throws YarnException { + public StopContainersResponse + stopContainers(StopContainersRequest requests) throws YarnException, + IOException { Exception e = new Exception("Dummy function", new Exception( "Dummy function cause")); throw new YarnException(e); } + + @Override + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException, IOException { + List list = new ArrayList(); + list.add(status); + GetContainerStatusesResponse response = + GetContainerStatusesResponse.newInstance(list, null); + return null; + } } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java index 68aeb56..76384d3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; import junit.framework.Assert; @@ -33,13 +35,14 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -122,9 +125,6 @@ private void test(String rpcClass) throws Exception { ApplicationAttemptId.newInstance(applicationId, 0); ContainerId containerId = ContainerId.newInstance(applicationAttemptId, 100); - StartContainerRequest scRequest = - recordFactory.newRecordInstance(StartContainerRequest.class); - scRequest.setContainerLaunchContext(containerLaunchContext); NodeId nodeId = NodeId.newInstance("localhost", 1234); Resource resource = Resource.newInstance(1234, 2); ContainerTokenIdentifier containerTokenIdentifier = @@ -132,22 +132,32 @@ private void test(String rpcClass) throws Exception { resource, System.currentTimeMillis() + 10000, 42, 42); Token containerToken = newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier); - scRequest.setContainerToken(containerToken); - proxy.startContainer(scRequest); - - GetContainerStatusRequest gcsRequest = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - gcsRequest.setContainerId(containerId); - GetContainerStatusResponse response = proxy.getContainerStatus(gcsRequest); - ContainerStatus status = response.getStatus(); - + + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + containerToken); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + proxy.startContainers(allRequests); + + List containerIds = new ArrayList(); + containerIds.add(containerId); + GetContainerStatusesRequest gcsRequest = + GetContainerStatusesRequest.newInstance(containerIds); + GetContainerStatusesResponse response = + proxy.getContainerStatuses(gcsRequest); + List statuses = response.getContainerStatuses(); + //test remote exception boolean exception = false; try { - StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class); - stopRequest.setContainerId(containerId); - proxy.stopContainer(stopRequest); - } catch (YarnException e) { + StopContainersRequest stopRequest = + recordFactory.newRecordInstance(StopContainersRequest.class); + stopRequest.setContainerIds(containerIds); + proxy.stopContainers(stopRequest); + } catch (YarnException e) { exception = true; Assert.assertTrue(e.getMessage().contains(EXCEPTION_MSG)); Assert.assertTrue(e.getMessage().contains(EXCEPTION_CAUSE)); @@ -158,46 +168,51 @@ private void test(String rpcClass) throws Exception { Assert.assertTrue(exception); server.stop(); - Assert.assertNotNull(status); - Assert.assertEquals(ContainerState.RUNNING, status.getState()); + Assert.assertNotNull(statuses.get(0)); + Assert.assertEquals(ContainerState.RUNNING, statuses.get(0).getState()); } public class DummyContainerManager implements ContainerManagementProtocol { - private ContainerStatus status = null; - + private List statuses = new ArrayList(); + @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException { - GetContainerStatusResponse response = - recordFactory.newRecordInstance(GetContainerStatusResponse.class); - response.setStatus(status); + GetContainerStatusesResponse response = + recordFactory.newRecordInstance(GetContainerStatusesResponse.class); + response.setContainerStatuses(statuses); return response; } @Override - public StartContainerResponse startContainer(StartContainerRequest request) - throws YarnException { - Token containerToken = request.getContainerToken(); - ContainerTokenIdentifier tokenId = null; - - try { - tokenId = newContainerTokenIdentifier(containerToken); - } catch (IOException e) { - throw RPCUtil.getRemoteException(e); + public StartContainersResponse startContainers( + StartContainersRequest requests) throws YarnException { + StartContainersResponse response = + recordFactory.newRecordInstance(StartContainersResponse.class); + for (StartContainerRequest request : requests.getStartContainerRequests()) { + Token containerToken = request.getContainerToken(); + ContainerTokenIdentifier tokenId = null; + + try { + tokenId = newContainerTokenIdentifier(containerToken); + } catch (IOException e) { + throw RPCUtil.getRemoteException(e); + } + ContainerStatus status = + recordFactory.newRecordInstance(ContainerStatus.class); + status.setState(ContainerState.RUNNING); + status.setContainerId(tokenId.getContainerID()); + status.setExitStatus(0); + statuses.add(status); + } - StartContainerResponse response = - recordFactory.newRecordInstance(StartContainerResponse.class); - status = recordFactory.newRecordInstance(ContainerStatus.class); - status.setState(ContainerState.RUNNING); - status.setContainerId(tokenId.getContainerID()); - status.setExitStatus(0); return response; } @Override - public StopContainerResponse stopContainer(StopContainerRequest request) + public StopContainersResponse stopContainers(StopContainersRequest request) throws YarnException { Exception e = new Exception(EXCEPTION_MSG, new Exception(EXCEPTION_CAUSE)); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/SerializedException.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/SerializedException.java deleted file mode 100644 index 9750607..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/SerializedException.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.records; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; - -@Private -@Unstable -public abstract class SerializedException { - - public abstract void init(String message, Throwable cause); - - public abstract void init(String message); - - public abstract void init(Throwable cause); - - public abstract String getMessage(); - - public abstract String getRemoteTrace(); - - public abstract SerializedException getCause(); -} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/SerializedExceptionPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/SerializedExceptionPBImpl.java deleted file mode 100644 index 428cfe4..0000000 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/SerializedExceptionPBImpl.java +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.records.impl.pb; - -import java.io.PrintWriter; -import java.io.StringWriter; - -import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto; -import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProtoOrBuilder; -import org.apache.hadoop.yarn.server.api.records.SerializedException; - -public class SerializedExceptionPBImpl extends SerializedException { - - SerializedExceptionProto proto = SerializedExceptionProto - .getDefaultInstance(); - SerializedExceptionProto.Builder builder = null; - boolean viaProto = false; - - public SerializedExceptionPBImpl() { - } - - public SerializedExceptionPBImpl(SerializedExceptionProto proto) { - this.proto = proto; - viaProto = true; - } - - private SerializedExceptionPBImpl(Throwable t) { - init(t); - } - - public void init(String message) { - maybeInitBuilder(); - builder.setMessage(message); - } - - public void init(Throwable t) { - maybeInitBuilder(); - if (t == null) { - return; - } - - if (t.getCause() == null) { - } else { - builder.setCause(new SerializedExceptionPBImpl(t.getCause()).getProto()); - builder.setClassName(t.getClass().getCanonicalName()); - } - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - t.printStackTrace(pw); - pw.close(); - if (sw.toString() != null) - builder.setTrace(sw.toString()); - if (t.getMessage() != null) - builder.setMessage(t.getMessage()); - } - - public void init(String message, Throwable t) { - init(t); - if (message != null) - builder.setMessage(message); - } - - @Override - public String getMessage() { - SerializedExceptionProtoOrBuilder p = viaProto ? proto : builder; - return p.getMessage(); - } - - @Override - public String getRemoteTrace() { - SerializedExceptionProtoOrBuilder p = viaProto ? proto : builder; - return p.getTrace(); - } - - @Override - public SerializedException getCause() { - SerializedExceptionProtoOrBuilder p = viaProto ? proto : builder; - if (p.hasCause()) { - return new SerializedExceptionPBImpl(p.getCause()); - } else { - return null; - } - } - - public SerializedExceptionProto getProto() { - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = SerializedExceptionProto.newBuilder(proto); - } - viaProto = false; - } -} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java index e472746..8bdff62 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java @@ -22,12 +22,12 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.NodeAction; -import org.apache.hadoop.yarn.server.api.records.SerializedException; import org.apache.hadoop.yarn.util.Records; /** @@ -59,10 +59,4 @@ public static NodeHeartbeatResponse newNodeHeartbeatResponse(int responseId, } return response; } - - public static SerializedException newSerializedException(Throwable e) { - SerializedException se = Records.newRecord(SerializedException.class); - se.init(e); - return se; - } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java index 41fc68e..f728eb3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NodeHeartbeatRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NodeHeartbeatResponsePBImpl; @@ -45,7 +46,6 @@ import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl; import org.apache.hadoop.yarn.server.api.records.impl.pb.NodeStatusPBImpl; -import org.apache.hadoop.yarn.server.api.records.impl.pb.SerializedExceptionPBImpl; import org.junit.Test; /** diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalResourceStatus.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalResourceStatus.java index 70a96cd..d3c5a8f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalResourceStatus.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalResourceStatus.java @@ -18,8 +18,8 @@ package org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords; import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.URL; -import org.apache.hadoop.yarn.server.api.records.SerializedException; public interface LocalResourceStatus { public LocalResource getResource(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalResourceStatusPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalResourceStatusPBImpl.java index 6ecb934..bc5fcd4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalResourceStatusPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalResourceStatusPBImpl.java @@ -18,9 +18,11 @@ package org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.impl.pb; import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase; +import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.URLPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto; import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto; @@ -28,8 +30,6 @@ import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalResourceStatusProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalResourceStatusProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.ResourceStatusTypeProto; -import org.apache.hadoop.yarn.server.api.records.SerializedException; -import org.apache.hadoop.yarn.server.api.records.impl.pb.SerializedExceptionPBImpl; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalResourceStatus; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 712bc43..583c0a3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -23,12 +23,12 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeMap; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; @@ -50,25 +50,25 @@ import org.apache.hadoop.service.ServiceStateChangeListener; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.InvalidContainerException; import org.apache.hadoop.yarn.exceptions.NMNotYetReadyException; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; @@ -130,8 +130,6 @@ private final NodeStatusUpdater nodeStatusUpdater; - private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); - protected LocalDirsHandlerService dirsHandler; protected final AsyncDispatcher dispatcher; private final ApplicationACLsManager aclsManager; @@ -316,18 +314,25 @@ protected NMTokenIdentifier selectNMTokenIdentifier( return resultId; } + protected void authorizeUser(UserGroupInformation remoteUgi, + NMTokenIdentifier nmTokenIdentifier) throws YarnException { + if (!remoteUgi.getUserName().equals( + nmTokenIdentifier.getApplicationAttemptId().toString())) { + throw RPCUtil.getRemoteException("Expected applicationAttemptId: " + + remoteUgi.getUserName() + "Found: " + + nmTokenIdentifier.getApplicationAttemptId()); + } + } + /** * @param containerTokenIdentifier * of the container to be started - * @param ugi - * ugi corresponding to the remote end making the api-call * @throws YarnException */ @Private @VisibleForTesting protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier, - ContainerTokenIdentifier containerTokenIdentifier, - UserGroupInformation ugi) throws YarnException { + ContainerTokenIdentifier containerTokenIdentifier) throws YarnException { ContainerId containerId = containerTokenIdentifier.getContainerID(); String containerIDStr = containerId.toString(); @@ -342,12 +347,6 @@ protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier, .append(" was used for starting container with container token") .append(" issued for application attempt : ") .append(containerId.getApplicationAttemptId()); - } else if (!ugi.getUserName().equals( - nmTokenIdentifier.getApplicationAttemptId().toString())) { - unauthorized = true; - messageBuilder.append("\nExpected applicationAttemptId: ") - .append(ugi.getUserName()).append(" Found: ") - .append(nmTokenIdentifier.getApplicationAttemptId().toString()); } else if (!this.context.getContainerTokenSecretManager() .isValidStartContainerRequest(containerTokenIdentifier)) { // Is the container being relaunched? Or RPC layer let startCall with @@ -363,7 +362,6 @@ protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier, .append(System.currentTimeMillis()).append(" found ") .append(containerTokenIdentifier.getExpiryTimeStamp()); } - if (unauthorized) { String msg = messageBuilder.toString(); LOG.error(msg); @@ -372,18 +370,48 @@ protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier, } /** - * Start a container on this NodeManager. + * Start a list of containers on this NodeManager. */ - @SuppressWarnings("unchecked") @Override - public StartContainerResponse startContainer(StartContainerRequest request) - throws YarnException, IOException { - + public StartContainersResponse + startContainers(StartContainersRequest requests) throws YarnException, + IOException { if (blockNewContainerRequests.get()) { throw new NMNotYetReadyException( "Rejecting new containers as NodeManager has not" + " yet connected with ResourceManager"); } + UserGroupInformation remoteUgi = getRemoteUgi(); + NMTokenIdentifier nmTokenIdentifier = selectNMTokenIdentifier(remoteUgi); + authorizeUser(remoteUgi,nmTokenIdentifier); + List succeededContainers = new ArrayList(); + Map failedContainers = + new HashMap(); + for (StartContainerRequest request : requests.getStartContainerRequests()) { + ContainerId containerId = null; + try { + ContainerTokenIdentifier containerTokenIdentifier = + verifyAndGetContainerTokenIdentifier(request.getContainerToken()); + containerId = containerTokenIdentifier.getContainerID(); + startContainerInternal(nmTokenIdentifier, containerTokenIdentifier, + request); + succeededContainers.add(containerId); + } catch (YarnException e) { + failedContainers.put(containerId, SerializedException.newInstance(e)); + } catch (InvalidToken ie) { + failedContainers.put(containerId, SerializedException.newInstance(ie)); + } + } + + return StartContainersResponse.newInstance(auxiliaryServices.getMetaData(), + succeededContainers, failedContainers); + } + + @SuppressWarnings("unchecked") + private void startContainerInternal(NMTokenIdentifier nmTokenIdentifier, + ContainerTokenIdentifier containerTokenIdentifier, + StartContainerRequest request) throws YarnException, InvalidToken { + /* * 1) It should save the NMToken into NMTokenSecretManager. This is done * here instead of RPC layer because at the time of opening/authenticating @@ -395,18 +423,8 @@ public StartContainerResponse startContainer(StartContainerRequest request) * belongs to correct Node Manager (part of retrieve password). c) It has * correct RMIdentifier. d) It is not expired. */ - // update NMToken - - UserGroupInformation remoteUgi = getRemoteUgi(); - NMTokenIdentifier nmTokenIdentifier = selectNMTokenIdentifier(remoteUgi); - - // Validate containerToken - ContainerTokenIdentifier containerTokenIdentifier = - verifyAndGetContainerTokenIdentifier(request.getContainerToken()); - - authorizeStartRequest(nmTokenIdentifier, containerTokenIdentifier, - remoteUgi); - + authorizeStartRequest(nmTokenIdentifier, containerTokenIdentifier); + if (containerTokenIdentifier.getRMIdentifer() != nodeStatusUpdater .getRMIdentifier()) { // Is the container coming from unknown RM @@ -415,9 +433,9 @@ public StartContainerResponse startContainer(StartContainerRequest request) .append(" rejected as it is allocated by a previous RM"); throw new InvalidContainerException(sb.toString()); } - + // update NMToken updateNMTokenIdentifier(nmTokenIdentifier); - + ContainerId containerId = containerTokenIdentifier.getContainerID(); String containerIdStr = containerId.toString(); String user = containerTokenIdentifier.getApplicationSubmitter(); @@ -461,14 +479,10 @@ public StartContainerResponse startContainer(StartContainerRequest request) containerTokenIdentifier); NMAuditLogger.logSuccess(user, AuditConstants.START_CONTAINER, "ContainerManageImpl", applicationID, containerId); - StartContainerResponse response = - recordFactory.newRecordInstance(StartContainerResponse.class); - response.setAllServicesMetaData(auxiliaryServices.getMetaData()); // TODO launchedContainer misplaced -> doesn't necessarily mean a container // launch. A finished Application will not launch containers. metrics.launchedContainer(); - metrics.allocateContainer(containerTokenIdentifier.getResource()); - return response; + metrics.allocateContainer(containerTokenIdentifier.getResource()); } protected ContainerTokenIdentifier verifyAndGetContainerTokenIdentifier( @@ -528,22 +542,37 @@ private Credentials parseCredentials(ContainerLaunchContext launchContext) } /** - * Stop the container running on this NodeManager. + * Stop a list of containers running on this NodeManager. */ @Override - @SuppressWarnings("unchecked") - public StopContainerResponse stopContainer(StopContainerRequest request) + public StopContainersResponse stopContainers(StopContainersRequest requests) throws YarnException, IOException { - ContainerId containerID = request.getContainerId(); + List succeededRequests = new ArrayList(); + Map failedRequests = + new HashMap(); + for (ContainerId id : requests.getContainerIds()) { + try { + stopContainerInternal(id); + succeededRequests.add(id); + } catch (YarnException e) { + failedRequests.put(id, SerializedException.newInstance(e)); + } catch (IOException ioe) { + throw ioe; + } + } + return StopContainersResponse + .newInstance(succeededRequests, failedRequests); + } + + @SuppressWarnings("unchecked") + private void stopContainerInternal(ContainerId containerID) + throws YarnException, IOException { String containerIDStr = containerID.toString(); Container container = this.context.getContainers().get(containerID); LOG.info("Getting container-status for " + containerIDStr); authorizeGetAndStopContainerRequest(containerID, container, true); - StopContainerResponse response = - recordFactory.newRecordInstance(StopContainerResponse.class); - dispatcher.getEventHandler().handle( new ContainerKillEvent(containerID, "Container killed by the ApplicationMaster.")); @@ -555,15 +584,34 @@ public StopContainerResponse stopContainer(StopContainerRequest request) // TODO: Move this code to appropriate place once kill_container is // implemented. nodeStatusUpdater.sendOutofBandHeartBeat(); - - return response; } + /** + * Get a list of container statuses running on this NodeManager + */ @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws YarnException, IOException { + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException, IOException { - ContainerId containerID = request.getContainerId(); + List succeededRequests = new ArrayList(); + Map failedRequests = + new HashMap(); + for (ContainerId id : request.getContainerIds()) { + try { + ContainerStatus status = getContainerStatusInternal(id); + succeededRequests.add(status); + } catch (YarnException e) { + failedRequests.put(id, SerializedException.newInstance(e)); + } catch (IOException ioe) { + throw ioe; + } + } + return GetContainerStatusesResponse.newInstance(succeededRequests, + failedRequests); + } + + private ContainerStatus getContainerStatusInternal(ContainerId containerID) + throws YarnException, IOException { String containerIDStr = containerID.toString(); Container container = this.context.getContainers().get(containerID); @@ -572,10 +620,7 @@ public GetContainerStatusResponse getContainerStatus( ContainerStatus containerStatus = container.cloneAndGetContainerStatus(); LOG.info("Returning " + containerStatus); - GetContainerStatusResponse response = - recordFactory.newRecordInstance(GetContainerStatusResponse.class); - response.setStatus(containerStatus); - return response; + return containerStatus; } @Private diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index c1f3eb9..f0cf9db 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -103,7 +103,7 @@ public ContainerImpl(Configuration conf, Dispatcher dispatcher, ContainerLaunchContext launchContext, Credentials creds, NodeManagerMetrics metrics, - ContainerTokenIdentifier containerTokenIdentifier) throws IOException { + ContainerTokenIdentifier containerTokenIdentifier) { this.daemonConf = conf; this.dispatcher = dispatcher; this.launchContext = launchContext; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index a55d6f0..66f21f6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -53,6 +53,7 @@ import org.apache.hadoop.util.DiskChecker; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -288,11 +289,10 @@ private LocalizerStatus createStatus() throws InterruptedException { stat.setStatus(ResourceStatusType.FETCH_SUCCESS); } catch (ExecutionException e) { stat.setStatus(ResourceStatusType.FETCH_FAILURE); - stat.setException( - YarnServerBuilderUtils.newSerializedException(e.getCause())); + stat.setException(SerializedException.newInstance(e.getCause())); } catch (CancellationException e) { stat.setStatus(ResourceStatusType.FETCH_FAILURE); - stat.setException(YarnServerBuilderUtils.newSerializedException(e)); + stat.setException(SerializedException.newInstance(e)); } // TODO shouldn't remove until ACK i.remove(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java index 76ef074..e6914af 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java @@ -195,8 +195,7 @@ public void setBlockNewContainerRequests(boolean blockNewContainerRequests) { @Override protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier, - ContainerTokenIdentifier containerTokenIdentifier, - UserGroupInformation ugi) throws YarnException { + ContainerTokenIdentifier containerTokenIdentifier) throws YarnException { // do nothing } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java index 13dc099..cc9b7d9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java @@ -146,7 +146,7 @@ public void testLocalFilesCleanup() throws InterruptedException, @Override public void testContainerLaunchFromPreviousRM() throws InterruptedException, - IOException { + IOException, YarnException { // Don't run the test if the binary is not available. if (!shouldRunTest()) { LOG.info("LCE binary path is not passed. Not running the test"); @@ -155,6 +155,29 @@ public void testContainerLaunchFromPreviousRM() throws InterruptedException, LOG.info("Running testContainerLaunchFromPreviousRM"); super.testContainerLaunchFromPreviousRM(); } + + @Override + public void testMultipleContainersLaunch() throws Exception { + // Don't run the test if the binary is not available. + if (!shouldRunTest()) { + LOG.info("LCE binary path is not passed. Not running the test"); + return; + } + LOG.info("Running testContainerLaunchFromPreviousRM"); + super.testMultipleContainersLaunch(); + } + + @Override + public void testMultipleContainersStopAndGetStatus() throws Exception { + // Don't run the test if the binary is not available. + if (!shouldRunTest()) { + LOG.info("LCE binary path is not passed. Not running the test"); + return; + } + LOG.info("Running testContainerLaunchFromPreviousRM"); + super.testMultipleContainersStopAndGetStatus(); + } + private boolean shouldRunTest() { return System .getProperty(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH) != null; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java index 294c93e..ba644ab 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java @@ -20,18 +20,19 @@ import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.Token; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; @@ -46,7 +47,6 @@ import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; -import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.Test; @@ -62,7 +62,7 @@ private static File remoteLogDir = new File("target", TestEventFlow.class.getName() + "-remoteLogDir").getAbsoluteFile(); private static final long SIMULATED_RM_IDENTIFIER = 1234; - + @Test public void testSuccessfulContainerLaunch() throws InterruptedException, IOException, YarnException { @@ -140,21 +140,25 @@ public long getRMIdentifier() { ContainerId cID = ContainerId.newInstance(applicationAttemptId, 0); String user = "testing"; - StartContainerRequest request = - recordFactory.newRecordInstance(StartContainerRequest.class); - request.setContainerLaunchContext(launchContext); - request.setContainerToken(TestContainerManager.createContainerToken(cID, - SIMULATED_RM_IDENTIFIER, context.getNodeId(), user, - context.getContainerTokenSecretManager())); - containerManager.startContainer(request); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(launchContext, + TestContainerManager.createContainerToken(cID, + SIMULATED_RM_IDENTIFIER, context.getNodeId(), user, + context.getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); BaseContainerManagerTest.waitForContainerState(containerManager, cID, ContainerState.RUNNING); - StopContainerRequest stopRequest = - recordFactory.newRecordInstance(StopContainerRequest.class); - stopRequest.setContainerId(cID); - containerManager.stopContainer(stopRequest); + List containerIds = new ArrayList(); + containerIds.add(cID); + StopContainersRequest stopRequest = + StopContainersRequest.newInstance(containerIds); + containerManager.stopContainers(stopRequest); BaseContainerManagerTest.waitForContainerState(containerManager, cID, ContainerState.COMPLETE); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java index e07a28d..fab9e01 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java @@ -39,8 +39,9 @@ import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -129,13 +130,17 @@ public void testClearLocalDirWhenNodeReboot() throws IOException, List commands = new ArrayList(); containerLaunchContext.setCommands(commands); - final StartContainerRequest startRequest = - Records.newRecord(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); NodeId nodeId = nm.getNMContext().getNodeId(); - startRequest.setContainerToken(TestContainerManager.createContainerToken( - cId, 0, nodeId, destinationFile, nm.getNMContext() - .getContainerTokenSecretManager())); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + TestContainerManager.createContainerToken( + cId, 0, nodeId, destinationFile, nm.getNMContext() + .getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(scRequest); + final StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + final UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(cId.getApplicationAttemptId() .toString()); @@ -145,16 +150,17 @@ public void testClearLocalDirWhenNodeReboot() throws IOException, currentUser.doAs(new PrivilegedExceptionAction() { @Override public Void run() throws YarnException, IOException { - nm.getContainerManager().startContainer(startRequest); + nm.getContainerManager().startContainers(allRequests); return null; } }); - GetContainerStatusRequest request = - Records.newRecord(GetContainerStatusRequest.class); - request.setContainerId(cId); + List containerIds = new ArrayList(); + containerIds.add(cId); + GetContainerStatusesRequest request = + GetContainerStatusesRequest.newInstance(containerIds); Container container = - nm.getNMContext().getContainers().get(request.getContainerId()); + nm.getNMContext().getContainers().get(request.getContainerIds().get(0)); final int MAX_TRIES = 20; int numTries = 0; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java index d01eb66..a05e341 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java @@ -20,6 +20,8 @@ import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CyclicBarrier; @@ -31,6 +33,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -282,15 +285,18 @@ public void run() { try { while (!isStopped && numContainers < 10) { ContainerId cId = TestNodeManagerShutdown.createContainerId(); - StartContainerRequest startRequest = - recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainerToken(null); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + null); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); System.out.println("no. of containers to be launched: " + numContainers); numContainers++; try { - getContainerManager().startContainer(startRequest); + getContainerManager().startContainers(allRequests); } catch (YarnException e) { numContainersRejected++; Assert.assertTrue(e.getMessage().contains( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java index fe3c5da..6fcb1e0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java @@ -25,6 +25,7 @@ import java.io.PrintWriter; import java.net.InetSocketAddress; import java.security.PrivilegedAction; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -40,8 +41,9 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -53,7 +55,6 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.URL; -import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -181,12 +182,6 @@ public static void startContainer(NodeManager nm, ContainerId cId, containerLaunchContext.setLocalResources(localResources); List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); - StartContainerRequest startRequest = - recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest - .setContainerToken(TestContainerManager.createContainerToken(cId, 0, - nodeId, user, nm.getNMContext().getContainerTokenSecretManager())); final InetSocketAddress containerManagerBindAddress = NetUtils.createSocketAddrForHost("127.0.0.1", 12345); UserGroupInformation currentUser = UserGroupInformation @@ -210,13 +205,22 @@ public ContainerManagementProtocol run() { containerManagerBindAddress, conf); } }); - containerManager.startContainer(startRequest); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + TestContainerManager.createContainerToken(cId, 0, + nodeId, user, nm.getNMContext().getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); - GetContainerStatusRequest request = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - request.setContainerId(cId); + List containerIds = new ArrayList(); + containerIds.add(cId); + GetContainerStatusesRequest request = + GetContainerStatusesRequest.newInstance(containerIds); ContainerStatus containerStatus = - containerManager.getContainerStatus(request).getStatus(); + containerManager.getContainerStatuses(request).getContainerStatuses().get(0); Assert.assertEquals(ContainerState.RUNNING, containerStatus.getState()); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java index 2a3e3d5..73bea03 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java @@ -45,8 +45,8 @@ import org.apache.hadoop.io.retry.RetryProxy; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.service.ServiceOperations; import org.apache.hadoop.service.Service.STATE; +import org.apache.hadoop.service.ServiceOperations; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -65,7 +65,6 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.ResourceTracker; -import org.apache.hadoop.yarn.server.api.ServerRMProxy; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java index ff9078c..d8f1979 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java @@ -21,16 +21,21 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import java.io.IOException; import java.net.URISyntaxException; import java.util.ArrayList; +import junit.framework.Assert; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalResourceStatusProto; @@ -80,7 +85,7 @@ static LocalResourceStatus createLocalResourceStatus() { e.setStackTrace(new StackTraceElement[] { new StackTraceElement("foo", "bar", "baz", 10), new StackTraceElement("sbb", "one", "onm", 10) }); - ret.setException(YarnServerBuilderUtils.newSerializedException(e)); + ret.setException(SerializedException.newInstance(e)); return ret; } @@ -176,4 +181,33 @@ public void testLocalizerHeartbeatResponseSerDe() throws Exception { assertEquals(createResource(), rsrcD.getResourceSpecs().get(0).getResource()); } + + @Test(timeout=10000) + public void testSerializedExceptionDeSer() throws Exception{ + // without cause + YarnException yarnEx = new YarnException("Yarn_Exception"); + SerializedException serEx = SerializedException.newInstance(yarnEx); + Throwable throwable = serEx.deSerialize(); + Assert.assertEquals(yarnEx.getClass(), throwable.getClass()); + Assert.assertEquals(yarnEx.getMessage(), throwable.getMessage()); + + // with cause + IOException ioe = new IOException("Test_IOException"); + RuntimeException runtimeException = + new RuntimeException("Test_RuntimeException", ioe); + YarnException yarnEx2 = + new YarnException("Test_YarnException", runtimeException); + + SerializedException serEx2 = SerializedException.newInstance(yarnEx2); + Throwable throwable2 = serEx2.deSerialize(); + throwable2.printStackTrace(); + Assert.assertEquals(yarnEx2.getClass(), throwable2.getClass()); + Assert.assertEquals(yarnEx2.getMessage(), throwable2.getMessage()); + + Assert.assertEquals(runtimeException.getClass(), throwable2.getCause().getClass()); + Assert.assertEquals(runtimeException.getMessage(), throwable2.getCause().getMessage()); + + Assert.assertEquals(ioe.getClass(), throwable2.getCause().getCause().getClass()); + Assert.assertEquals(ioe.getMessage(), throwable2.getCause().getCause().getMessage()); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java index cfcf7f6..809aa26 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java @@ -20,6 +20,8 @@ import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import junit.framework.Assert; @@ -32,7 +34,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -192,12 +194,15 @@ protected void authorizeGetAndStopContainerRequest(ContainerId containerId, Container container, boolean stopRequest) throws YarnException { // do nothing } - + @Override + protected void authorizeUser(UserGroupInformation remoteUgi, + NMTokenIdentifier nmTokenIdentifier) { + // do nothing + } @Override protected void authorizeStartRequest( NMTokenIdentifier nmTokenIdentifier, - ContainerTokenIdentifier containerTokenIdentifier, - UserGroupInformation ugi) throws YarnException { + ContainerTokenIdentifier containerTokenIdentifier) throws YarnException { // do nothing } @@ -238,18 +243,20 @@ public static void waitForContainerState(ContainerManagementProtocol containerMa public static void waitForContainerState(ContainerManagementProtocol containerManager, ContainerId containerID, ContainerState finalState, int timeOutMax) throws InterruptedException, YarnException, IOException { - GetContainerStatusRequest request = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - request.setContainerId(containerID); - ContainerStatus containerStatus = - containerManager.getContainerStatus(request).getStatus(); - int timeoutSecs = 0; + List list = new ArrayList(); + list.add(containerID); + GetContainerStatusesRequest request = + GetContainerStatusesRequest.newInstance(list); + ContainerStatus containerStatus = + containerManager.getContainerStatuses(request).getContainerStatuses() + .get(0); + int timeoutSecs = 0; while (!containerStatus.getState().equals(finalState) && timeoutSecs++ < timeOutMax) { Thread.sleep(1000); LOG.info("Waiting for container to get into state " + finalState + ". Current state is " + containerStatus.getState()); - containerStatus = containerManager.getContainerStatus(request).getStatus(); + containerStatus = containerManager.getContainerStatuses(request).getContainerStatuses().get(0); } LOG.info("Container state is " + containerStatus.getState()); Assert.assertEquals("ContainerState is not correct (timedout)", diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java index fdd0ed4..73ce3fc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.net.InetAddress; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -37,9 +38,13 @@ import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Shell; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -51,6 +56,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.exceptions.InvalidContainerException; @@ -63,6 +69,7 @@ import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager; @@ -88,11 +95,11 @@ public void setup() throws IOException { super.setup(); } - private ContainerId createContainerId() { + private ContainerId createContainerId(int id) { ApplicationId appId = ApplicationId.newInstance(0, 0); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1); - ContainerId containerId = ContainerId.newInstance(appAttemptId, 0); + ContainerId containerId = ContainerId.newInstance(appAttemptId, id); return containerId; } @@ -119,6 +126,14 @@ protected UserGroupInformation getRemoteUgi() throws YarnException { .getKeyId())); return ugi; } + + @Override + protected void authorizeGetAndStopContainerRequest(ContainerId containerId, + Container container, boolean stopRequest) throws YarnException { + if(container == null || container.getUser().equals("Fail")){ + throw new YarnException("Reject this container"); + } + } }; } @@ -138,12 +153,17 @@ public void testContainerManagerInitialization() throws IOException { // Just do a query for a non-existing container. boolean throwsException = false; try { - GetContainerStatusRequest request = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - ContainerId cId = createContainerId(); - request.setContainerId(cId); - containerManager.getContainerStatus(request); - } catch (YarnException e) { + List containerIds = new ArrayList(); + ContainerId id =createContainerId(0); + containerIds.add(id); + GetContainerStatusesRequest request = + GetContainerStatusesRequest.newInstance(containerIds); + GetContainerStatusesResponse response = + containerManager.getContainerStatuses(request); + if(response.getFailedRequests().containsKey(id)){ + throw response.getFailedRequests().get(id).deSerialize(); + } + } catch (Throwable e) { throwsException = true; } Assert.assertTrue(throwsException); @@ -163,7 +183,7 @@ public void testContainerSetup() throws Exception { fileWriter.close(); // ////// Construct the Container-id - ContainerId cId = createContainerId(); + ContainerId cId = createContainerId(0); // ////// Construct the container-spec. ContainerLaunchContext containerLaunchContext = @@ -182,14 +202,17 @@ public void testContainerSetup() throws Exception { new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - StartContainerRequest startRequest = - recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainerToken(createContainerToken(cId, - DUMMY_RM_IDENTIFIER, context.getNodeId(), user, - context.getContainerTokenSecretManager())); - containerManager.startContainer(startRequest); + StartContainerRequest scRequest = + StartContainerRequest.newInstance( + containerLaunchContext, + createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), + user, context.getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE); @@ -237,7 +260,7 @@ public void testContainerLaunchAndStop() throws IOException, new File(tmpDir, "start_file.txt").getAbsoluteFile(); // ////// Construct the Container-id - ContainerId cId = createContainerId(); + ContainerId cId = createContainerId(0); if (Shell.WINDOWS) { fileWriter.println("@echo Hello World!> " + processStartFile); @@ -272,13 +295,17 @@ public void testContainerLaunchAndStop() throws IOException, List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); - StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainerToken(createContainerToken(cId, - DUMMY_RM_IDENTIFIER, context.getNodeId(), user, - context.getContainerTokenSecretManager())); - containerManager.startContainer(startRequest); - + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + createContainerToken(cId, + DUMMY_RM_IDENTIFIER, context.getNodeId(), user, + context.getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); + int timeoutSecs = 0; while (!processStartFile.exists() && timeoutSecs++ < 20) { Thread.sleep(1000); @@ -305,18 +332,18 @@ public void testContainerLaunchAndStop() throws IOException, Assert.assertTrue("Process is not alive!", DefaultContainerExecutor.containerIsAlive(pid)); - StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class); - stopRequest.setContainerId(cId); - containerManager.stopContainer(stopRequest); - + List containerIds = new ArrayList(); + containerIds.add(cId); + StopContainersRequest stopRequest = + StopContainersRequest.newInstance(containerIds); + containerManager.stopContainers(stopRequest); BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE); - GetContainerStatusRequest gcsRequest = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - gcsRequest.setContainerId(cId); + GetContainerStatusesRequest gcsRequest = + GetContainerStatusesRequest.newInstance(containerIds); ContainerStatus containerStatus = - containerManager.getContainerStatus(gcsRequest).getStatus(); + containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0); int expectedExitCode = Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() : ExitCode.TERMINATED.getExitCode(); Assert.assertEquals(expectedExitCode, containerStatus.getExitStatus()); @@ -325,7 +352,7 @@ public void testContainerLaunchAndStop() throws IOException, Assert.assertFalse("Process is still alive!", DefaultContainerExecutor.containerIsAlive(pid)); } - + private void testContainerLaunchAndExit(int exitCode) throws IOException, InterruptedException, YarnException { @@ -335,7 +362,7 @@ private void testContainerLaunchAndExit(int exitCode) throws IOException, new File(tmpDir, "start_file.txt").getAbsoluteFile(); // ////// Construct the Container-id - ContainerId cId = createContainerId(); + ContainerId cId = createContainerId(0); if (Shell.WINDOWS) { fileWriter.println("@echo Hello World!> " + processStartFile); @@ -376,21 +403,26 @@ private void testContainerLaunchAndExit(int exitCode) throws IOException, List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); - StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainerToken(createContainerToken(cId, - DUMMY_RM_IDENTIFIER, context.getNodeId(), user, - context.getContainerTokenSecretManager())); - containerManager.startContainer(startRequest); + StartContainerRequest scRequest = + StartContainerRequest.newInstance( + containerLaunchContext, + createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), + user, context.getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE); - GetContainerStatusRequest gcsRequest = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - gcsRequest.setContainerId(cId); + List containerIds = new ArrayList(); + containerIds.add(cId); + GetContainerStatusesRequest gcsRequest = + GetContainerStatusesRequest.newInstance(containerIds); ContainerStatus containerStatus = - containerManager.getContainerStatus(gcsRequest).getStatus(); + containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0); // Verify exit status matches exit state of script Assert.assertEquals(exitCode, @@ -439,7 +471,7 @@ public void testLocalFilesCleanup() throws InterruptedException, fileWriter.close(); // ////// Construct the Container-id - ContainerId cId = createContainerId(); + ContainerId cId = createContainerId(0); ApplicationId appId = cId.getApplicationAttemptId().getApplicationId(); // ////// Construct the container-spec. @@ -460,11 +492,17 @@ public void testLocalFilesCleanup() throws InterruptedException, new HashMap(); localResources.put(destinationFile, rsrc_alpha); containerLaunchContext.setLocalResources(localResources); - StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class); - request.setContainerLaunchContext(containerLaunchContext); - request.setContainerToken(createContainerToken(cId, DUMMY_RM_IDENTIFIER, - context.getNodeId(), user, context.getContainerTokenSecretManager())); - containerManager.startContainer(request); + + StartContainerRequest scRequest = + StartContainerRequest.newInstance( + containerLaunchContext, + createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), + user, context.getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE); @@ -528,29 +566,37 @@ public void testLocalFilesCleanup() throws InterruptedException, @Test public void testContainerLaunchFromPreviousRM() throws IOException, - InterruptedException { + InterruptedException, YarnException { containerManager.start(); ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); - ContainerId cId1 = createContainerId(); - ContainerId cId2 = createContainerId(); + ContainerId cId1 = createContainerId(0); + ContainerId cId2 = createContainerId(0); containerLaunchContext .setLocalResources(new HashMap()); // Construct the Container with Invalid RMIdentifier StartContainerRequest startRequest1 = - recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest1.setContainerLaunchContext(containerLaunchContext); + StartContainerRequest.newInstance(containerLaunchContext, + createContainerToken(cId1, + ResourceManagerConstants.RM_INVALID_IDENTIFIER, context.getNodeId(), + user, context.getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(startRequest1); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); - startRequest1.setContainerToken(createContainerToken(cId1, - ResourceManagerConstants.RM_INVALID_IDENTIFIER, context.getNodeId(), - user, context.getContainerTokenSecretManager())); boolean catchException = false; try { - containerManager.startContainer(startRequest1); - } catch (YarnException e) { + StartContainersResponse response = containerManager.startContainers(allRequests); + if(response.getFailedRequests().containsKey(cId1)) { + throw response.getFailedRequests().get(cId1).deSerialize(); + } + } catch (Throwable e) { + e.printStackTrace(); catchException = true; Assert.assertTrue(e.getMessage().contains( "Container " + cId1 + " rejected as it is allocated by a previous RM")); @@ -563,21 +609,143 @@ public void testContainerLaunchFromPreviousRM() throws IOException, // Construct the Container with a RMIdentifier within current RM StartContainerRequest startRequest2 = - recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest2.setContainerLaunchContext(containerLaunchContext); - startRequest2.setContainerToken(createContainerToken(cId2, - DUMMY_RM_IDENTIFIER, context.getNodeId(), user, - context.getContainerTokenSecretManager())); + StartContainerRequest.newInstance(containerLaunchContext, + createContainerToken(cId2, + DUMMY_RM_IDENTIFIER, context.getNodeId(), user, + context.getContainerTokenSecretManager())); + List list2 = new ArrayList(); + list.add(startRequest2); + StartContainersRequest allRequests2 = + StartContainersRequest.newInstance(list2); + containerManager.startContainers(allRequests2); + boolean noException = true; try { - containerManager.startContainer(startRequest2); + containerManager.startContainers(allRequests2); } catch (YarnException e) { noException = false; } // Verify that startContainer get no YarnException Assert.assertTrue(noException); } - + + @Test + public void testMultipleContainersLaunch() throws Exception { + containerManager.start(); + + List list = new ArrayList(); + ContainerLaunchContext containerLaunchContext = + recordFactory.newRecordInstance(ContainerLaunchContext.class); + for (int i = 0; i < 10; i++) { + ContainerId cId = createContainerId(i); + long identifier = 0; + if ((i & 1) == 0) + // container with even id fail + identifier = ResourceManagerConstants.RM_INVALID_IDENTIFIER; + else + identifier = DUMMY_RM_IDENTIFIER; + Token containerToken = + createContainerToken(cId, identifier, context.getNodeId(), user, + context.getContainerTokenSecretManager()); + StartContainerRequest request = + StartContainerRequest.newInstance(containerLaunchContext, + containerToken); + list.add(request); + } + StartContainersRequest requestList = + StartContainersRequest.newInstance(list); + + StartContainersResponse response = + containerManager.startContainers(requestList); + + Assert.assertEquals(5, response.getSuccessfullyStartedContainers().size()); + for (ContainerId id : response.getSuccessfullyStartedContainers()) { + // Containers with odd id should succeed. + Assert.assertEquals(1, id.getId() & 1); + } + Assert.assertEquals(5, response.getFailedRequests().size()); + for (Map.Entry entry : response + .getFailedRequests().entrySet()) { + // Containers with even id should fail. + Assert.assertEquals(0, entry.getKey().getId() & 1); + Assert.assertTrue(entry.getValue().getMessage() + .contains( + "Container " + entry.getKey() + " rejected as it is allocated by a previous RM")); + } + } + + @Test + public void testMultipleContainersStopAndGetStatus() throws Exception { + containerManager.start(); + List startRequest = + new ArrayList(); + ContainerLaunchContext containerLaunchContext = + recordFactory.newRecordInstance(ContainerLaunchContext.class); + + List containerIds = new ArrayList(); + for (int i = 0; i < 10; i++) { + ContainerId cId = createContainerId(i); + String user = null; + if ((i & 1) == 0) { + // container with even id fail + user = "Fail"; + } else { + user = "Pass"; + } + Token containerToken = + createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), + user, context.getContainerTokenSecretManager()); + StartContainerRequest request = + StartContainerRequest.newInstance(containerLaunchContext, + containerToken); + startRequest.add(request); + containerIds.add(cId); + } + // start containers + StartContainersRequest requestList = + StartContainersRequest.newInstance(startRequest); + containerManager.startContainers(requestList); + + // Get container statuses + GetContainerStatusesRequest statusRequest = + GetContainerStatusesRequest.newInstance(containerIds); + GetContainerStatusesResponse statusResponse = + containerManager.getContainerStatuses(statusRequest); + Assert.assertEquals(5, statusResponse.getContainerStatuses().size()); + for (ContainerStatus status : statusResponse.getContainerStatuses()) { + // Containers with odd id should succeed + Assert.assertEquals(1, status.getContainerId().getId() & 1); + } + Assert.assertEquals(5, statusResponse.getFailedRequests().size()); + for (Map.Entry entry : statusResponse + .getFailedRequests().entrySet()) { + // Containers with even id should fail. + Assert.assertEquals(0, entry.getKey().getId() & 1); + Assert.assertTrue(entry.getValue().getMessage() + .contains("Reject this container")); + } + + // stop containers + StopContainersRequest stopRequest = + StopContainersRequest.newInstance(containerIds); + StopContainersResponse stopResponse = + containerManager.stopContainers(stopRequest); + Assert.assertEquals(5, stopResponse.getSuccessfullyStoppedContainers() + .size()); + for (ContainerId id : stopResponse.getSuccessfullyStoppedContainers()) { + // Containers with odd id should succeed. + Assert.assertEquals(1, id.getId() & 1); + } + Assert.assertEquals(5, stopResponse.getFailedRequests().size()); + for (Map.Entry entry : stopResponse + .getFailedRequests().entrySet()) { + // Containers with even id should fail. + Assert.assertEquals(0, entry.getKey().getId() & 1); + Assert.assertTrue(entry.getValue().getMessage() + .contains("Reject this container")); + } + } + public static Token createContainerToken(ContainerId cId, long rmIdentifier, NodeId nodeId, String user, NMContainerTokenSecretManager containerTokenSecretManager) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index 9e17c47..ee6217c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -42,9 +42,10 @@ import org.apache.hadoop.util.Shell.ExitCodeException; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -399,10 +400,14 @@ public void testContainerEnvVariables() throws Exception { // set up the rest of the container List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); - StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainerToken(createContainerToken(cId)); - containerManager.startContainer(startRequest); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + createContainerToken(cId)); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); int timeoutSecs = 0; while (!processStartFile.exists() && timeoutSecs++ < 20) { @@ -465,18 +470,20 @@ public void testContainerEnvVariables() throws Exception { Assert.assertTrue("Process is not alive!", DefaultContainerExecutor.containerIsAlive(pid)); - StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class); - stopRequest.setContainerId(cId); - containerManager.stopContainer(stopRequest); + // Now test the stop functionality. + List containerIds = new ArrayList(); + containerIds.add(cId); + StopContainersRequest stopRequest = + StopContainersRequest.newInstance(containerIds); + containerManager.stopContainers(stopRequest); BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE); - GetContainerStatusRequest gcsRequest = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - gcsRequest.setContainerId(cId); + GetContainerStatusesRequest gcsRequest = + GetContainerStatusesRequest.newInstance(containerIds); ContainerStatus containerStatus = - containerManager.getContainerStatus(gcsRequest).getStatus(); + containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0); int expectedExitCode = Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() : ExitCode.TERMINATED.getExitCode(); Assert.assertEquals(expectedExitCode, containerStatus.getExitStatus()); @@ -544,11 +551,15 @@ public void testDelayedKill() throws Exception { List commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile)); containerLaunchContext.setCommands(commands); Token containerToken = createContainerToken(cId); - StartContainerRequest startRequest = - recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainerToken(containerToken); - containerManager.startContainer(startRequest); + + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + containerToken); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); int timeoutSecs = 0; while (!processStartFile.exists() && timeoutSecs++ < 20) { @@ -559,19 +570,22 @@ public void testDelayedKill() throws Exception { processStartFile.exists()); // Now test the stop functionality. - StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class); - stopRequest.setContainerId(cId); - containerManager.stopContainer(stopRequest); + List containerIds = new ArrayList(); + containerIds.add(cId); + StopContainersRequest stopRequest = + StopContainersRequest.newInstance(containerIds); + containerManager.stopContainers(stopRequest); BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE); // container stop sends a sigterm followed by a sigkill - GetContainerStatusRequest gcsRequest = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - gcsRequest.setContainerId(cId); + GetContainerStatusesRequest gcsRequest = + GetContainerStatusesRequest.newInstance(containerIds); + ContainerStatus containerStatus = - containerManager.getContainerStatus(gcsRequest).getStatus(); + containerManager.getContainerStatuses(gcsRequest) + .getContainerStatuses().get(0); Assert.assertEquals(ExitCode.FORCE_KILLED.getExitCode(), containerStatus.getExitStatus()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalResourceStatus.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalResourceStatus.java index 882068a..1eeab7c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalResourceStatus.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalResourceStatus.java @@ -18,8 +18,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer; import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.URL; -import org.apache.hadoop.yarn.server.api.records.SerializedException; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalResourceStatus; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index d4832e0..26578bf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -80,6 +80,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; @@ -1013,8 +1014,8 @@ private LocalizerStatus createLocalizerStatusForFailedResource( String localizerId, LocalResourceRequest req) { LocalizerStatus status = createLocalizerStatus(localizerId); LocalResourceStatus resourceStatus = new LocalResourceStatusPBImpl(); - resourceStatus.setException(YarnServerBuilderUtils - .newSerializedException(new YarnException("test"))); + resourceStatus.setException(SerializedException + .newInstance(new YarnException("test"))); resourceStatus.setStatus(ResourceStatusType.FETCH_FAILURE); resourceStatus.setResource(req); status.addResourceStatus(resourceStatus); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 152b988..5179f3f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -63,6 +63,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -809,14 +810,18 @@ public void testLogAggregationForRealContainerLaunch() throws IOException, commands.add("/bin/bash"); commands.add(scriptFile.getAbsolutePath()); containerLaunchContext.setCommands(commands); - StartContainerRequest startRequest = - recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); - startRequest.setContainerToken(TestContainerManager.createContainerToken( - cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, - context.getContainerTokenSecretManager())); - this.containerManager.startContainer(startRequest); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + TestContainerManager.createContainerToken( + cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, + context.getContainerTokenSecretManager())); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + this.containerManager.startContainers(allRequests); + BaseContainerManagerTest.waitForContainerState(this.containerManager, cId, ContainerState.COMPLETE); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index cd73fab..e0a4bfe 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -40,8 +40,9 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -227,9 +228,6 @@ public void testContainerKillOnMemoryOverflow() throws IOException, commands.add(scriptFile.getAbsolutePath()); containerLaunchContext.setCommands(commands); Resource r = BuilderUtils.newResource(8 * 1024 * 1024, 1); - StartContainerRequest startRequest = - recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(containerLaunchContext); ContainerTokenIdentifier containerIdentifier = new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user, r, System.currentTimeMillis() + 120000, 123, DUMMY_RM_IDENTIFIER); @@ -237,8 +235,14 @@ public void testContainerKillOnMemoryOverflow() throws IOException, BuilderUtils.newContainerToken(context.getNodeId(), containerManager.getContext().getContainerTokenSecretManager() .createPassword(containerIdentifier), containerIdentifier); - startRequest.setContainerToken(containerToken); - containerManager.startContainer(startRequest); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(containerLaunchContext, + containerToken); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + containerManager.startContainers(allRequests); int timeoutSecs = 0; while (!processStartFile.exists() && timeoutSecs++ < 20) { @@ -260,11 +264,12 @@ public void testContainerKillOnMemoryOverflow() throws IOException, BaseContainerManagerTest.waitForContainerState(containerManager, cId, ContainerState.COMPLETE, 60); - GetContainerStatusRequest gcsRequest = - recordFactory.newRecordInstance(GetContainerStatusRequest.class); - gcsRequest.setContainerId(cId); + List containerIds = new ArrayList(); + containerIds.add(cId); + GetContainerStatusesRequest gcsRequest = + GetContainerStatusesRequest.newInstance(containerIds); ContainerStatus containerStatus = - containerManager.getContainerStatus(gcsRequest).getStatus(); + containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0); Assert.assertEquals(ExitCode.TERMINATED.getExitCode(), containerStatus.getExitStatus()); String expectedMsgPattern = diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index c1f02da..a1272ce 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -22,6 +22,8 @@ import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; @@ -37,7 +39,10 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; @@ -46,8 +51,6 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.factories.RecordFactory; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -68,8 +71,6 @@ private final RMAppAttempt application; private final Configuration conf; - private final RecordFactory recordFactory = - RecordFactoryProvider.getRecordFactory(null); private final AMLauncherEventType eventType; private final RMContext rmContext; private final Container masterContainer; @@ -102,22 +103,41 @@ private void launch() throws IOException, YarnException { + " for AM " + application.getAppAttemptId()); ContainerLaunchContext launchContext = createAMContainerLaunchContext(applicationContext, masterContainerID); - StartContainerRequest request = - recordFactory.newRecordInstance(StartContainerRequest.class); - request.setContainerLaunchContext(launchContext); - request.setContainerToken(masterContainer.getContainerToken()); - containerMgrProxy.startContainer(request); - LOG.info("Done launching container " + masterContainer - + " for AM " + application.getAppAttemptId()); + + StartContainerRequest scRequest = + StartContainerRequest.newInstance(launchContext, + masterContainer.getContainerToken()); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + + StartContainersResponse response = + containerMgrProxy.startContainers(allRequests); + if (response.getFailedRequests() != null + && response.getFailedRequests().containsKey(masterContainerID)) { + throw (YarnException) response.getFailedRequests().get(masterContainerID) + .deSerialize(); + } else { + LOG.info("Done launching container " + masterContainer + " for AM " + + application.getAppAttemptId()); + } } private void cleanup() throws IOException, YarnException { connect(); ContainerId containerId = masterContainer.getId(); - StopContainerRequest stopRequest = - recordFactory.newRecordInstance(StopContainerRequest.class); - stopRequest.setContainerId(containerId); - containerMgrProxy.stopContainer(stopRequest); + List containerIds = new ArrayList(); + containerIds.add(containerId); + StopContainersRequest stopRequest = + StopContainersRequest.newInstance(containerIds); + StopContainersResponse response = + containerMgrProxy.stopContainers(stopRequest); + if (response.getFailedRequests() != null + && response.getFailedRequests().containsKey(containerId)) { + throw (YarnException) response.getFailedRequests().get(containerId) + .deSerialize(); + } } // Protected. For tests. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index 4d6cbf2..3c76f0b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -35,7 +35,8 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -212,9 +213,11 @@ public synchronized void finishTask(Task task) throws IOException, NodeManager nodeManager = task.getNodeManager(); ContainerId containerId = task.getContainerId(); task.stop(); - StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class); - stopRequest.setContainerId(containerId); - nodeManager.stopContainer(stopRequest); + List containerIds = new ArrayList(); + containerIds.add(containerId); + StopContainersRequest stopRequest = + StopContainersRequest.newInstance(containerIds); + nodeManager.stopContainers(stopRequest); Resources.subtractFrom(used, requestSpec.get(task.getPriority())); @@ -339,10 +342,15 @@ private synchronized void assign(Priority priority, NodeType type, updateResourceRequests(requests.get(priority), type, task); // Launch the container - StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class); - startRequest.setContainerLaunchContext(createCLC()); - startRequest.setContainerToken(container.getContainerToken()); - nodeManager.startContainer(startRequest); + StartContainerRequest scRequest = + StartContainerRequest.newInstance(createCLC(), + container.getContainerToken()); + List list = + new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); + nodeManager.startContainers(allRequests); break; } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java index 7772b04..2c9d678 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java @@ -31,12 +31,13 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -158,61 +159,60 @@ public void heartbeat() throws IOException, YarnException { } @Override - synchronized public StartContainerResponse startContainer( - StartContainerRequest request) + synchronized public StartContainersResponse startContainers( + StartContainersRequest requests) throws YarnException { - Token containerToken = request.getContainerToken(); - ContainerTokenIdentifier tokenId = null; + for (StartContainerRequest request : requests.getStartContainerRequests()) { + Token containerToken = request.getContainerToken(); + ContainerTokenIdentifier tokenId = null; - try { - tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken); - } catch (IOException e) { - throw RPCUtil.getRemoteException(e); - } + try { + tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken); + } catch (IOException e) { + throw RPCUtil.getRemoteException(e); + } - ContainerId containerID = tokenId.getContainerID(); - ApplicationId applicationId = - containerID.getApplicationAttemptId().getApplicationId(); + ContainerId containerID = tokenId.getContainerID(); + ApplicationId applicationId = + containerID.getApplicationAttemptId().getApplicationId(); - List applicationContainers = containers.get(applicationId); - if (applicationContainers == null) { - applicationContainers = new ArrayList(); - containers.put(applicationId, applicationContainers); - } - - // Sanity check - for (Container container : applicationContainers) { - if (container.getId().compareTo(containerID) - == 0) { - throw new IllegalStateException( - "Container " + containerID + - " already setup on node " + containerManagerAddress); + List applicationContainers = containers.get(applicationId); + if (applicationContainers == null) { + applicationContainers = new ArrayList(); + containers.put(applicationId, applicationContainers); + } + + // Sanity check + for (Container container : applicationContainers) { + if (container.getId().compareTo(containerID) == 0) { + throw new IllegalStateException("Container " + containerID + + " already setup on node " + containerManagerAddress); + } } - } - Container container = - BuilderUtils.newContainer(containerID, - this.nodeId, nodeHttpAddress, - tokenId.getResource(), - null, null // DKDC - Doesn't matter + Container container = + BuilderUtils.newContainer(containerID, this.nodeId, nodeHttpAddress, + tokenId.getResource(), null, null // DKDC - Doesn't matter ); - ContainerStatus containerStatus = - BuilderUtils.newContainerStatus(container.getId(), ContainerState.NEW, - "", -1000); - applicationContainers.add(container); - containerStatusMap.put(container, containerStatus); - Resources.subtractFrom(available, tokenId.getResource()); - Resources.addTo(used, tokenId.getResource()); - - if(LOG.isDebugEnabled()) { - LOG.debug("startContainer:" + " node=" + containerManagerAddress - + " application=" + applicationId + " container=" + container - + " available=" + available + " used=" + used); - } + ContainerStatus containerStatus = + BuilderUtils.newContainerStatus(container.getId(), + ContainerState.NEW, "", -1000); + applicationContainers.add(container); + containerStatusMap.put(container, containerStatus); + Resources.subtractFrom(available, tokenId.getResource()); + Resources.addTo(used, tokenId.getResource()); + + if (LOG.isDebugEnabled()) { + LOG.debug("startContainer:" + " node=" + containerManagerAddress + + " application=" + applicationId + " container=" + container + + " available=" + available + " used=" + used); + } - StartContainerResponse response = recordFactory.newRecordInstance(StartContainerResponse.class); + } + StartContainersResponse response = + StartContainersResponse.newInstance(null, null, null); return response; } @@ -225,76 +225,79 @@ synchronized public void checkResourceUsage() { } @Override - synchronized public StopContainerResponse stopContainer(StopContainerRequest request) + synchronized public StopContainersResponse stopContainers(StopContainersRequest request) throws YarnException { - ContainerId containerID = request.getContainerId(); - String applicationId = String.valueOf( - containerID.getApplicationAttemptId().getApplicationId().getId()); - - // Mark the container as COMPLETE - List applicationContainers = containers.get(applicationId); - for (Container c : applicationContainers) { - if (c.getId().compareTo(containerID) == 0) { - ContainerStatus containerStatus = containerStatusMap.get(c); - containerStatus.setState(ContainerState.COMPLETE); - containerStatusMap.put(c, containerStatus); + for (ContainerId containerID : request.getContainerIds()) { + String applicationId = + String.valueOf(containerID.getApplicationAttemptId() + .getApplicationId().getId()); + + // Mark the container as COMPLETE + List applicationContainers = containers.get(applicationId); + for (Container c : applicationContainers) { + if (c.getId().compareTo(containerID) == 0) { + ContainerStatus containerStatus = containerStatusMap.get(c); + containerStatus.setState(ContainerState.COMPLETE); + containerStatusMap.put(c, containerStatus); + } } - } - - // Send a heartbeat - try { - heartbeat(); - } catch (IOException ioe) { - throw RPCUtil.getRemoteException(ioe); - } - - // Remove container and update status - int ctr = 0; - Container container = null; - for (Iterator i=applicationContainers.iterator(); i.hasNext();) { - container = i.next(); - if (container.getId().compareTo(containerID) == 0) { - i.remove(); - ++ctr; + + // Send a heartbeat + try { + heartbeat(); + } catch (IOException ioe) { + throw RPCUtil.getRemoteException(ioe); } - } - - if (ctr != 1) { - throw new IllegalStateException("Container " + containerID + - " stopped " + ctr + " times!"); - } - - Resources.addTo(available, container.getResource()); - Resources.subtractFrom(used, container.getResource()); - if(LOG.isDebugEnabled()) { - LOG.debug("stopContainer:" + " node=" + containerManagerAddress - + " application=" + applicationId + " container=" + containerID - + " available=" + available + " used=" + used); - } + // Remove container and update status + int ctr = 0; + Container container = null; + for (Iterator i = applicationContainers.iterator(); i + .hasNext();) { + container = i.next(); + if (container.getId().compareTo(containerID) == 0) { + i.remove(); + ++ctr; + } + } - StopContainerResponse response = recordFactory.newRecordInstance(StopContainerResponse.class); - return response; + if (ctr != 1) { + throw new IllegalStateException("Container " + containerID + + " stopped " + ctr + " times!"); + } + + Resources.addTo(available, container.getResource()); + Resources.subtractFrom(used, container.getResource()); + + if (LOG.isDebugEnabled()) { + LOG.debug("stopContainer:" + " node=" + containerManagerAddress + + " application=" + applicationId + " container=" + containerID + + " available=" + available + " used=" + used); + } + } + return StopContainersResponse.newInstance(null,null); } @Override - synchronized public GetContainerStatusResponse getContainerStatus(GetContainerStatusRequest request) throws YarnException { - ContainerId containerId = request.getContainerId(); - List appContainers = - containers.get( - containerId.getApplicationAttemptId().getApplicationId()); - Container container = null; - for (Container c : appContainers) { - if (c.getId().equals(containerId)) { - container = c; + synchronized public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException { + List statuses = new ArrayList(); + for (ContainerId containerId : request.getContainerIds()) { + List appContainers = + containers.get(containerId.getApplicationAttemptId() + .getApplicationId()); + Container container = null; + for (Container c : appContainers) { + if (c.getId().equals(containerId)) { + container = c; + } + } + if (container != null + && containerStatusMap.get(container).getState() != null) { + statuses.add(containerStatusMap.get(container)); } } - GetContainerStatusResponse response = - recordFactory.newRecordInstance(GetContainerStatusResponse.class); - if (container != null && containerStatusMap.get(container).getState() != null) { - response.setStatus(containerStatusMap.get(container)); - } - return response; + return GetContainerStatusesResponse.newInstance(statuses, null); } public static org.apache.hadoop.yarn.server.api.records.NodeStatus diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java index 7bae0c1..af07fb8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java @@ -36,14 +36,14 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -98,23 +98,23 @@ public MyContainerManager() { } @Override - public StartContainerResponse - startContainer(StartContainerRequest request) + public StartContainersResponse + startContainers(StartContainersRequest request) throws YarnException { - containerTokens = request.getContainerLaunchContext().getTokens(); - return null; + containerTokens = request.getStartContainerRequests().get(0).getContainerLaunchContext().getTokens(); + return StartContainersResponse.newInstance(null, null, null); } @Override - public StopContainerResponse stopContainer(StopContainerRequest request) + public StopContainersResponse stopContainers(StopContainersRequest request) throws YarnException { - return null; + return StopContainersResponse.newInstance(null, null); } @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws YarnException { - return null; + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException { + return GetContainerStatusesResponse.newInstance(null, null); } public Credentials getContainerCredentials() throws IOException { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java index ad55b0c..27dbe86 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java @@ -19,7 +19,10 @@ package org.apache.hadoop.yarn.server.resourcemanager; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.commons.logging.Log; @@ -27,16 +30,18 @@ import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest; -import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -69,9 +74,10 @@ int maxAppAttempts; @Override - public StartContainerResponse - startContainer(StartContainerRequest request) + public StartContainersResponse + startContainers(StartContainersRequest requests) throws YarnException { + StartContainerRequest request = requests.getStartContainerRequests().get(0); LOG.info("Container started by MyContainerManager: " + request); launched = true; Map env = @@ -95,11 +101,13 @@ Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV)); maxAppAttempts = Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV)); - return null; + return StartContainersResponse.newInstance( + new HashMap(), new ArrayList(), + new HashMap()); } @Override - public StopContainerResponse stopContainer(StopContainerRequest request) + public StopContainersResponse stopContainers(StopContainersRequest request) throws YarnException { LOG.info("Container cleaned up by MyContainerManager"); cleanedup = true; @@ -107,11 +115,10 @@ public StopContainerResponse stopContainer(StopContainerRequest request) } @Override - public GetContainerStatusResponse getContainerStatus( - GetContainerStatusRequest request) throws YarnException { + public GetContainerStatusesResponse getContainerStatuses( + GetContainerStatusesRequest request) throws YarnException { return null; } - } @Test diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java index 0492471..fc2fda8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java @@ -19,7 +19,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.security; import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.IOException; import java.lang.annotation.Annotation; @@ -50,6 +52,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.event.Dispatcher; @@ -158,6 +162,9 @@ public void testClientToAMs() throws Exception { ContainerManagementProtocol containerManager = mock(ContainerManagementProtocol.class); + StartContainersResponse mockResponse = mock(StartContainersResponse.class); + when(containerManager.startContainers((StartContainersRequest) any())) + .thenReturn(mockResponse); final DrainDispatcher dispatcher = new DrainDispatcher(); MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java index 7781d50..34a00ac 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java @@ -23,6 +23,8 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.List; import junit.framework.Assert; @@ -33,14 +35,18 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.ContainerManagementProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.SerializedException; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -328,9 +334,10 @@ protected String testStartContainer(YarnRPC rpc, ContainerId containerId, ApplicationAttemptId appAttemptId, NodeId nodeId, boolean isExceptionExpected) throws Exception { - GetContainerStatusRequest request = - Records.newRecord(GetContainerStatusRequest.class); - request.setContainerId(containerId); + List containerIds = new ArrayList(); + containerIds.add(containerId); + GetContainerStatusesRequest request = + GetContainerStatusesRequest.newInstance(containerIds); ContainerManagementProtocol proxy = null; @@ -338,8 +345,10 @@ protected String testStartContainer(YarnRPC rpc, proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, appAttemptId.toString()); - proxy.getContainerStatus(request); - + GetContainerStatusesResponse statuses = proxy.getContainerStatuses(request); + for(SerializedException ex : statuses.getFailedRequests().values()){ + throw (YarnException)ex.deSerialize(); + } } finally { if (proxy != null) { rpc.stopProxy(proxy, conf); @@ -352,17 +361,21 @@ private void startContainer(final YarnRPC rpc, org.apache.hadoop.yarn.api.records.Token containerToken, NodeId nodeId, String user) throws Exception { - StartContainerRequest request = - Records.newRecord(StartContainerRequest.class); - request.setContainerToken(containerToken); ContainerLaunchContext context = Records.newRecord(ContainerLaunchContext.class); - request.setContainerLaunchContext(context); - + StartContainerRequest scRequest = + StartContainerRequest.newInstance(context,containerToken); + List list = new ArrayList(); + list.add(scRequest); + StartContainersRequest allRequests = + StartContainersRequest.newInstance(list); ContainerManagementProtocol proxy = null; try { proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user); - proxy.startContainer(request); + StartContainersResponse response = proxy.startContainers(allRequests); + for(SerializedException ex : response.getFailedRequests().values()){ + throw (YarnException)ex.deSerialize(); + } } finally { if (proxy != null) { rpc.stopProxy(proxy, conf);