From 1d6a2ea1855ab5adf27f9eb39029dc32240fdc11 Mon Sep 17 00:00:00 2001 From: Rohith Sharma K S Date: Sun, 20 Nov 2016 14:44:05 +0530 Subject: [PATCH] YARN-5649 --- .../yarn/server/resourcemanager/RMAuditLogger.java | 1 + .../webapp/JAXBContextResolver.java | 3 +- .../resourcemanager/webapp/RMWebServices.java | 179 ++++++++++++++++----- .../server/resourcemanager/webapp/dao/AppInfo.java | 14 ++ .../resourcemanager/webapp/dao/AppTimeoutInfo.java | 68 ++++++++ .../webapp/dao/AppTimeoutsInfo.java | 45 ++++++ .../webapp/TestRMWebServicesApps.java | 2 +- .../webapp/TestRMWebServicesAppsModification.java | 119 +++++++++++++- 8 files changed, 386 insertions(+), 45 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutsInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index d52e002..051d979 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -66,6 +66,7 @@ "Update Application Priority"; public static final String UPDATE_APP_TIMEOUTS = "Update Application Timeouts"; + public static final String GET_APP_TIMEOUTS = "Get Application Timeouts"; public static final String CHANGE_CONTAINER_RESOURCE = "AM Changed Container Resource"; public static final String SIGNAL_CONTAINER = "Signal Container Request"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index b61072d..2f50a24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -54,7 +54,8 @@ public JAXBContextResolver() throws Exception { CapacitySchedulerQueueInfoList.class, ResourceInfo.class, UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class, StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class, - FairSchedulerQueueInfoList.class}; + FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class, + AppTimeoutInfo.class }; // these dao classes need root unwrapping final Class[] rootUnwrappedTypes = { NewApplication.class, ApplicationSubmissionContextInfo.class, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 2c61339..7c8c928 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -96,10 +97,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationPriorityRequest; +import org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -140,47 +143,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LogAggregationContextInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplication; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewReservation; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntry; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntryList; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDefinitionInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDeleteRequestInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDeleteResponseInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationListInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationSubmissionRequestInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateResponseInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.*; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -189,6 +151,7 @@ import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo; import org.apache.hadoop.yarn.util.AdHocLogDumper; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.ForbiddenException; import org.apache.hadoop.yarn.webapp.NotFoundException; @@ -2428,4 +2391,138 @@ public ReservationListResponse run() throws IOException, return Response.status(Status.OK).entity(resResponse).build(); } + @GET + @Path("/apps/{appid}/timeout/{type}") + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + public AppTimeoutInfo getAppTimeout(@Context HttpServletRequest hsr, + @PathParam("appid") String appId, @PathParam("type") String type) + throws AuthorizationException { + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + String userName = "UNKNOWN-USER"; + if (callerUGI != null) { + userName = callerUGI.getUserName(); + } + + ApplicationTimeoutType appTimeoutType = parseTimeoutType(type); + + RMApp app = null; + try { + app = getRMAppForAppId(appId); + } catch (NotFoundException e) { + RMAuditLogger.logFailure(userName, AuditConstants.GET_APP_TIMEOUTS, + "UNKNOWN", "RMWebService", + "Trying to get timeouts of an absent application " + appId); + throw e; + } + + Long timeoutValue = app.getApplicationTimeouts().get(appTimeoutType); + AppTimeoutInfo timeouts = + constructAppTimeoutDao(appTimeoutType, timeoutValue); + return timeouts; + } + + private ApplicationTimeoutType parseTimeoutType(String type) { + try { + // enum string is in the uppercase + return ApplicationTimeoutType + .valueOf(StringUtils.toUpperCase(type.trim())); + } catch (RuntimeException e) { + ApplicationTimeoutType[] typeArray = ApplicationTimeoutType.values(); + String allAppTimeoutTypes = Arrays.toString(typeArray); + throw new BadRequestException("Invalid application-state " + type.trim() + + " specified. It should be one of " + allAppTimeoutTypes); + } + } + + private AppTimeoutInfo constructAppTimeoutDao(ApplicationTimeoutType type, + Long timeoutInMillis) { + AppTimeoutInfo timeout = new AppTimeoutInfo(); + timeout.setTimeoutType(type); + if (timeoutInMillis != null) { + timeout.setExpireTime(Times.formatISO8601(timeoutInMillis.longValue())); + timeout.setRemainingTime( + Math.max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0)); + } + return timeout; + } + + @PUT + @Path("/apps/{appid}/timeout") + @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, + MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response updateApplicationTimeout(AppTimeoutInfo appTimeouts, + @Context HttpServletRequest hsr, @PathParam("appid") String appId) + throws AuthorizationException, YarnException, InterruptedException, + IOException { + init(); + if (appTimeouts == null) { + throw new YarnException("AppTimeouts cannot be null"); + } + + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); + if (callerUGI == null) { + throw new AuthorizationException( + "Unable to obtain user name, user not authenticated"); + } + + if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) { + return Response.status(Status.FORBIDDEN) + .entity("The default static user cannot carry out this operation.") + .build(); + } + + String userName = callerUGI.getUserName(); + RMApp app = null; + try { + app = getRMAppForAppId(appId); + } catch (NotFoundException e) { + RMAuditLogger.logFailure(userName, AuditConstants.UPDATE_APP_TIMEOUTS, + "UNKNOWN", "RMWebService", + "Trying to move an absent application " + appId); + throw e; + } + + return updateApplicationTimeouts(app, callerUGI, appTimeouts); + } + + private Response updateApplicationTimeouts(final RMApp app, + UserGroupInformation callerUGI, final AppTimeoutInfo appTimeout) + throws IOException, InterruptedException { + String userName = callerUGI.getUserName(); + try { + callerUGI.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws IOException, YarnException { + UpdateApplicationTimeoutsRequest request = + UpdateApplicationTimeoutsRequest + .newInstance(app.getApplicationId(), Collections.singletonMap( + appTimeout.getTimeoutType(), appTimeout.getExpireTime())); + rm.getClientRMService().updateApplicationTimeouts(request); + return null; + } + }); + } catch (UndeclaredThrowableException ue) { + // if the root cause is a permissions issue + // bubble that up to the user + if (ue.getCause() instanceof YarnException) { + YarnException ye = (YarnException) ue.getCause(); + if (ye.getCause() instanceof AccessControlException) { + String appId = app.getApplicationId().toString(); + String msg = "Unauthorized attempt to change timeout of app " + appId + + " by remote user " + userName; + return Response.status(Status.FORBIDDEN).entity(msg).build(); + } else { + throw ue; + } + } else { + throw ue; + } + } + AppTimeoutInfo timeout = constructAppTimeoutDao(appTimeout.getTimeoutType(), + app.getApplicationTimeouts().get(appTimeout.getTimeoutType())); + return Response.status(Status.OK).entity(timeout).build(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java index 42449b0..edc6688 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -29,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.LogAggregationStatus; @@ -114,6 +116,7 @@ protected String amNodeLabelExpression; protected ResourcesInfo resourceInfo = null; + protected AppTimeoutsInfo timeouts = new AppTimeoutsInfo(); public AppInfo() { } // JAXB needs this @@ -239,6 +242,17 @@ public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess, : null; } } + + for (Map.Entry entry : app + .getApplicationTimeouts().entrySet()) { + AppTimeoutInfo timeout = new AppTimeoutInfo(); + timeout.setTimeoutType(entry.getKey()); + long timeoutInMillis = entry.getValue().longValue(); + timeout.setExpireTime(Times.formatISO8601(timeoutInMillis)); + timeout.setRemainingTime( + Math.max((timeoutInMillis - System.currentTimeMillis()) / 1000, 0)); + timeouts.add(timeout); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java new file mode 100644 index 0000000..083eeb4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutInfo.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; + +@XmlRootElement(name = "timeout") +@XmlAccessorType(XmlAccessType.FIELD) +public class AppTimeoutInfo { + + @XmlElement(name = "type") + private ApplicationTimeoutType timeoutType; + + @XmlElement(name = "expiretime") + private String expireTime; + + @XmlElement(name = "remainingtimeinsec") + private long remainingTimeInSec; + + public AppTimeoutInfo() { + expireTime = "N/A"; + remainingTimeInSec = -1; + } + + public ApplicationTimeoutType getTimeoutType() { + return timeoutType; + } + + public String getExpireTime() { + return expireTime; + } + + public long getRemainingTimeInSec() { + return remainingTimeInSec; + } + + public void setTimeoutType(ApplicationTimeoutType type) { + this.timeoutType = type; + } + + public void setExpireTime(String expireTime) { + this.expireTime = expireTime; + } + + public void setRemainingTime(long remainingTime) { + this.remainingTimeInSec = remainingTime; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutsInfo.java new file mode 100644 index 0000000..f27118b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppTimeoutsInfo.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +@XmlRootElement(name = "timeouts") +@XmlAccessorType(XmlAccessType.FIELD) +public class AppTimeoutsInfo { + + @XmlElement(name = "timeout") + protected ArrayList timeouts = + new ArrayList(); + + public AppTimeoutsInfo() { + } // JAXB needs this + + public void add(AppTimeoutInfo timeoutInfo) { + timeouts.add(timeoutInfo); + } + + public ArrayList getAppTimeouts() { + return timeouts; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index fffa55b..30f25e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1419,7 +1419,7 @@ public void verifyAppsXML(NodeList nodes, RMApp app) public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException, Exception { - int expectedNumberOfElements = 34; + int expectedNumberOfElements = 35; String appNodeLabelExpression = null; String amNodeLabelExpression = null; if (app.getApplicationSubmissionContext() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java index e884a88..5fbcd60 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java @@ -61,6 +61,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; @@ -83,15 +84,17 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LogAggregationContextInfo; -import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Times; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; import org.apache.hadoop.yarn.webapp.JerseyTestBase; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; +import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.junit.After; @@ -117,6 +120,7 @@ import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.api.client.filter.LoggingFilter; +import com.sun.jersey.api.json.JSONConfiguration; import com.sun.jersey.api.json.JSONJAXBContext; import com.sun.jersey.api.json.JSONMarshaller; import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; @@ -1052,7 +1056,7 @@ public void testGetAppQueue() throws Exception { rm.stop(); } - @Test(timeout = 90000) + @Test public void testUpdateAppPriority() throws Exception { client().addFilter(new LoggingFilter(System.out)); @@ -1295,4 +1299,115 @@ protected static void verifyAppPriorityXML(ClientResponse response, assertEquals(queue, responseQueue); } + @Test + public void testUpdateAppTimeout() throws Exception { + client().addFilter(new LoggingFilter(System.out)); + + rm.start(); + rm.registerNode("127.0.0.1:1234", 2048); + String[] mediaTypes = + { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }; + MediaType[] contentTypes = + { MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE }; + for (String mediaType : mediaTypes) { + for (MediaType contentType : contentTypes) { + // application submitted without timeout + RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName); + + AppTimeoutInfo timeoutUpdate = new AppTimeoutInfo(); + long timeOutFromNow = 60; + String expireTime = Times + .formatISO8601(System.currentTimeMillis() + timeOutFromNow * 1000); + timeoutUpdate.setTimeoutType(ApplicationTimeoutType.LIFETIME); + timeoutUpdate.setExpireTime(expireTime); + + Object entity; + if (contentType.equals(MediaType.APPLICATION_JSON_TYPE)) { + entity = appTimeoutToJSON(timeoutUpdate); + } else { + entity = timeoutUpdate; + } + ClientResponse response = this + .constructWebResource("apps", app.getApplicationId().toString(), + "timeout") + .entity(entity, contentType).accept(mediaType) + .put(ClientResponse.class); + + if (!isAuthenticationEnabled()) { + assertResponseStatusCode(Status.UNAUTHORIZED, + response.getStatusInfo()); + continue; + } + assertResponseStatusCode(Status.OK, response.getStatusInfo()); + if (mediaType.contains(MediaType.APPLICATION_JSON)) { + verifyAppTimeoutJson(response, ApplicationTimeoutType.LIFETIME, + expireTime, timeOutFromNow); + } else { + verifyAppTimeoutXML(response, ApplicationTimeoutType.LIFETIME, + expireTime, timeOutFromNow); + } + + // invoke get + response = + this.constructWebResource("apps", app.getApplicationId().toString(), + "timeout", ApplicationTimeoutType.LIFETIME.toString()) + .accept(mediaType).get(ClientResponse.class); + assertResponseStatusCode(Status.OK, response.getStatusInfo()); + if (mediaType.contains(MediaType.APPLICATION_JSON)) { + verifyAppTimeoutJson(response, ApplicationTimeoutType.LIFETIME, + expireTime, timeOutFromNow); + } + } + } + rm.stop(); + } + + protected static void verifyAppTimeoutJson(ClientResponse response, + ApplicationTimeoutType type, String expireTime, long timeOutFromNow) + throws JSONException { + assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + JSONObject jsonTimeout = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, jsonTimeout.length()); + JSONObject json = jsonTimeout.getJSONObject("timeout"); + assertEquals("incorrect number of elements", 3, json.length()); + assertEquals(type.toString(), json.getString("type")); + assertEquals(expireTime, json.getString("expiretime")); + assertTrue(json.getLong("remainingtimeinsec") < timeOutFromNow); + } + + protected static void verifyAppTimeoutXML(ClientResponse response, + ApplicationTimeoutType type, String expireTime, long timeOutFromNow) + throws ParserConfigurationException, IOException, SAXException { + assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8, + response.getType().toString()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("timeout"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + Element element = (Element) nodes.item(0); + assertEquals(type.toString(), + WebServicesTestUtils.getXmlString(element, "type")); + assertEquals(expireTime, + WebServicesTestUtils.getXmlString(element, "expiretime")); + assertTrue(WebServicesTestUtils.getXmlLong(element, + "remainingtimeinsec") < timeOutFromNow); + } + + protected static String appTimeoutToJSON(AppTimeoutInfo timeout) + throws Exception { + StringWriter sw = new StringWriter(); + JSONJAXBContext ctx = new JSONJAXBContext( + JSONConfiguration.natural().rootUnwrapping(false).build(), + AppTimeoutInfo.class); + JSONMarshaller jm = ctx.createJSONMarshaller(); + jm.marshallToJSON(timeout, sw); + jm.marshallToJSON(timeout, System.out); + return sw.toString(); + } + } -- 2.7.4 (Apple Git-66)