diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 60fbfcd..39dcac4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -19,6 +19,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import java.io.IOException; +import java.nio.ByteBuffer; +import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Collection; import java.util.EnumSet; @@ -31,31 +33,51 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.Consumes; import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; @@ -70,6 +92,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppSubmissionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; @@ -77,6 +100,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplicationInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; @@ -84,12 +109,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.inject.Inject; import com.google.inject.Singleton; +import com.sun.jersey.core.util.Base64; @Singleton @Path("/ws/v1/cluster") @@ -590,4 +617,284 @@ public AppAttemptsInfo getAppAttempts(@PathParam("appid") String appId) { return appAttemptsInfo; } + + // can't return POJO because we can't control the status code + // it's always set to 200 when we need to allow it to be set + // to 202 + + @PUT + @Path("/apps/{appid}/kill") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response killApp(@Context HttpServletRequest hsr, + @PathParam("appid") String appId) throws AuthorizationException, + YarnException, InterruptedException, IOException { + + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr); + if(callerUGI == null) { + throw new AuthorizationException("Unable to obtain user name, user not authenticated"); + } + String userName = callerUGI.getUserName(); + RMApp app = null; + try { + app = appIdToRMApp(appId); + } + catch(NotFoundException e) { + RMAuditLogger.logFailure(userName, + AuditConstants.KILL_APP_REQUEST, "UNKNOWN", "RMWebService", + "Trying to kill an absent application " + appId); + throw e; + } + + if(app != null && !hasAppAcess(app, callerUGI, ApplicationAccessType.MODIFY_APP)) { + String msg = "Unauthorized attempt to kill appid " + + appId + "by remote user " + userName; + RMAuditLogger.logFailure(userName, AuditConstants.KILL_APP_REQUEST, + "UNKNOWN", "RMWebService", "Unauthorized attempt to kill appid " + appId); + return Response.status(Status.FORBIDDEN).entity(msg).build(); + } + + final ApplicationId appid = app.getApplicationId(); + KillApplicationResponse resp = callerUGI.doAs(new PrivilegedExceptionAction() { + @Override + public KillApplicationResponse run() throws IOException, YarnException { + KillApplicationRequest req = recordFactory.newRecordInstance(KillApplicationRequest.class); + req.setApplicationId(appid); + return rm.getClientRMService().forceKillApplication(req); + } + }); + + AppInfo appInfo = new AppInfo(app, hasAccess(app, hsr), hsr.getScheme() + "://"); + + if (resp.getIsKillCompleted()) { + RMAuditLogger.logSuccess(userName, AuditConstants.KILL_APP_REQUEST, + "RMWebService", app.getApplicationId()); + } else { + StringBuffer url = hsr.getRequestURL(); + String appUrl = url.substring(0, url.length() - "/kill".length()); + return Response.status(Status.ACCEPTED) + .entity(appInfo) + .header("Location", appUrl).build(); + } + + return Response.status(Status.OK).entity(new AppInfo(app, hasAccess(app, hsr), + hsr.getScheme() + "://")).build(); + } + + private RMApp appIdToRMApp(String appId) { + + if(appId == null || appId.isEmpty()) { + throw new NotFoundException("appId, " + appId + ", is empty or null"); + } + ApplicationId id; + try { + id = ConverterUtils.toApplicationId(recordFactory, appId); + } + catch(NumberFormatException e) { + throw new NotFoundException("appId is invalid"); + } + if(id == null) { + throw new NotFoundException("appId is invalid"); + } + RMApp app = rm.getRMContext().getRMApps().get(id); + if(app == null) { + throw new NotFoundException("app with id: " + appId + " not found"); + } + return app; + } + + private UserGroupInformation getCallerUserGroupInformation(HttpServletRequest hsr) { + + String remoteUser = hsr.getRemoteUser(); + UserGroupInformation callerUGI = null; + if (remoteUser != null) { + callerUGI = UserGroupInformation.createRemoteUser(remoteUser); + } + + return callerUGI; + } + + protected Boolean hasAppAcess(RMApp app, UserGroupInformation callerUGI, ApplicationAccessType type) { + if (callerUGI != null) { + if(!this.aclsManager.checkAccess(callerUGI, type, + app.getUser(), app.getApplicationId())) { + return false; + } + } + return true; + } + + protected Boolean hasQueueAcess(RMApp app, UserGroupInformation callerUGI, QueueACL type) { + if (callerUGI != null) { + if(!this.queueACLsManager.checkAccess(callerUGI, type, + app.getQueue())) { + return false; + } + } + return true; + } + + //reuse the code in ClientRMService to create new app + @POST + @Path("/app/id") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public NewApplicationInfo createNewApplication(@Context HttpServletRequest hsr) + throws AuthorizationException{ + + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr); + if(callerUGI == null) { + throw new AuthorizationException("Unable to obtain user name, user not authenticated"); + } + GetNewApplicationRequest req = recordFactory. + newRecordInstance(GetNewApplicationRequest.class); + GetNewApplicationResponse resp; + try { + resp = rm.getClientRMService().getNewApplication(req); + } + catch(YarnException e) { + String msg = "Unable to create new app from RM web service"; + LOG.error(msg, e); + throw new YarnRuntimeException(msg, e); + } + return new NewApplicationInfo(resp.getApplicationId(), resp.getMaximumResourceCapability()); + } + + @POST + @Path("/app/{appid}") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response submitNewApplication(AppSubmissionInfo newApp, @Context HttpServletRequest hsr, + @PathParam("appid") String appId) throws AuthorizationException, IOException, InterruptedException{ + + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr); + if(callerUGI == null) { + throw new AuthorizationException("Unable to obtain user name, user not authenticated"); + } + + RMApp app = null; + try { + app = appIdToRMApp(appId); + } + catch(NotFoundException e) { + // do nothing this is the expected response + ; + } + if(app != null) { + throw new BadRequestException("App with appId '" + appId + "' already exists"); + } + + if(appId.equals(newApp.getApplicationId()) == false) { + throw new BadRequestException("appid in url and submitted info don't match"); + } + if(newApp.getApplicationName().isEmpty()) { + throw new BadRequestException("Application name missing"); + } + if(newApp.getResource().getMemory() == 0 || newApp.getResource().getvCores() == 0) { + throw new BadRequestException("Requested resource memory and/or cores are 0"); + } + + ApplicationSubmissionContext appContext = createAppSubmissionContext(newApp); + final SubmitApplicationRequest req = recordFactory.newRecordInstance(SubmitApplicationRequest.class); + req.setApplicationSubmissionContext(appContext); + + + callerUGI.doAs(new + PrivilegedExceptionAction() { + @Override + public SubmitApplicationResponse run() throws IOException, YarnException { + return rm.getClientRMService().submitApplication(req); + } + }); + + StringBuffer url = hsr.getRequestURL(); + return Response.status(Status.ACCEPTED) + .header("Location", url).build(); + } + + protected ApplicationSubmissionContext createAppSubmissionContext(AppSubmissionInfo newApp) throws BadRequestException { + + // create local resources and app submission context + ApplicationSubmissionContext appContext = + recordFactory.newRecordInstance(ApplicationSubmissionContext.class); + appContext.setApplicationId(ConverterUtils.toApplicationId(recordFactory, newApp.getApplicationId())); + appContext.setApplicationName(newApp.getApplicationName()); + appContext.setQueue(newApp.getQueue()); + Priority priority = Records.newRecord(Priority.class); + priority.setPriority(newApp.getPriority()); + appContext.setPriority(priority); + appContext.setUnmanagedAM(newApp.isUnmanagedAM()); + appContext.setCancelTokensWhenComplete(newApp.isCancelTokensWhenComplete()); + appContext.setMaxAppAttempts(newApp.getMaxAppAttempts()); + appContext.setApplicationType(newApp.getApplicationType()); + appContext.setKeepContainersAcrossApplicationAttempts(newApp.isKeepContainers()); + appContext.setResource(createAppSubmissionContextResource(newApp)); + appContext.setAMContainerSpec(createContainerLaunchContext(newApp)); + return appContext; + } + + protected Resource createAppSubmissionContextResource(AppSubmissionInfo newApp) throws BadRequestException { + if(newApp.getResource().getvCores() > + rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)) { + throw new BadRequestException("Requested more cores than configured max"); + } + if(newApp.getResource().getMemory() > + rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)) { + throw new BadRequestException("Requested more memory than configured max"); + } + Resource r = recordFactory.newRecordInstance(Resource.class); + r.setMemory(newApp.getResource().getMemory()); + r.setVirtualCores(newApp.getResource().getvCores()); + return r; + } + + protected ContainerLaunchContext createContainerLaunchContext(AppSubmissionInfo newApp) throws BadRequestException { + + // create container launch context + ContainerLaunchContext ctx = recordFactory.newRecordInstance(ContainerLaunchContext.class); + ctx.setCommands(newApp.getContainerLaunchContext().getCommands()); + ctx.setApplicationACLs(newApp.getContainerLaunchContext().getAcls()); + ctx.setEnvironment(newApp.getContainerLaunchContext().getEnvironment()); + if(newApp.getContainerLaunchContext().getTokens().isEmpty() == false) { + ctx.getTokens().put(Base64.decode(newApp.getContainerLaunchContext().getTokens())); + } + HashMap hmap = new HashMap(); + for(Map.Entry entry: newApp.getContainerLaunchContext().getServiceData().entrySet()) { + if(entry.getValue().isEmpty() == false) { + hmap.put(entry.getKey(), ByteBuffer.wrap(Base64.decode(entry.getValue()))); + } + } + ctx.setServiceData(hmap); + // verify local resource values + if(newApp.getContainerLaunchContext().getLocalResources().size() <= 0) { + throw new BadRequestException("Request must specify at least one local resource"); + } + HashMap hlr = new HashMap(); + for(Map.Entry entry: newApp.getContainerLaunchContext().getLocalResources().entrySet()) { + LocalResourceInfo l = entry.getValue(); + if(l.getUrl() == null) { + throw new BadRequestException("File URI not set or was set to an incorrect " + + "value in the LocalResource for '" + entry.getKey() + "'"); + } + if(l.getType() == null) { + throw new BadRequestException("File type not set or was set to an incorrect " + + "value in the LocalResource for '" + entry.getKey() + "'"); + } + if(l.getVisibility() == null) { + throw new BadRequestException("File visibility not set or was set to an incorrect " + + "value in the LocalResource for '" + entry.getKey() + "'"); + } + LocalResource lr = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(l.getUrl()), + l.getType(), l.getVisibility(), l.getSize(), l.getTimestamp()); + hlr.put(entry.getKey(), lr); + } + ctx.setLocalResources(hlr); + ctx.setApplicationACLs(newApp.getContainerLaunchContext().getAcls()); + return ctx; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppSubmissionInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppSubmissionInfo.java new file mode 100644 index 0000000..3150baf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppSubmissionInfo.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.Priority; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class AppSubmissionInfo { + + String applicationId; + String applicationName; + String queue; + int priority; + ContainerLaunchInfo containerLaunchContext; + boolean isUnmanagedAM; + boolean cancelTokensWhenComplete; + int maxAppAttempts; + ResourceInfo resource; + String applicationType; + boolean keepContainers; + + public AppSubmissionInfo() { + applicationId = new String(); + applicationName = new String(); + containerLaunchContext = new ContainerLaunchInfo(); + resource = new ResourceInfo(); + priority = Priority.UNDEFINED.getPriority(); + isUnmanagedAM = false; + } + + public String getApplicationId() { + return applicationId; + } + + public String getApplicationName() { + return applicationName; + } + public String getQueue() { + return queue; + } + public int getPriority() { + return priority; + } + public ContainerLaunchInfo getContainerLaunchContext() { + return containerLaunchContext; + } + public boolean isUnmanagedAM() { + return isUnmanagedAM; + } + public boolean isCancelTokensWhenComplete() { + return cancelTokensWhenComplete; + } + public int getMaxAppAttempts() { + return maxAppAttempts; + } + public ResourceInfo getResource() { + return resource; + } + public String getApplicationType() { + return applicationType; + } + public boolean isKeepContainers() { + return keepContainers; + } + public void setApplicationId(String applicationId) { + this.applicationId = applicationId; + } + public void setApplicationName(String applicationName) { + this.applicationName = applicationName; + } + public void setQueue(String queue) { + this.queue = queue; + } + public void setPriority(int priority) { + this.priority = priority; + } + public void setContainerLaunchContext(ContainerLaunchInfo containerLaunchContext) { + this.containerLaunchContext = containerLaunchContext; + } + public void setUnmanagedAM(boolean isUnmanagedAM) { + this.isUnmanagedAM = isUnmanagedAM; + } + public void setCancelTokensWhenComplete(boolean cancelTokensWhenComplete) { + this.cancelTokensWhenComplete = cancelTokensWhenComplete; + } + public void setMaxAppAttempts(int maxAppAttempts) { + this.maxAppAttempts = maxAppAttempts; + } + public void setResource(ResourceInfo resource) { + this.resource = resource; + } + public void setApplicationType(String applicationType) { + this.applicationType = applicationType; + } + public void setKeepContainers(boolean keepContainers) { + this.keepContainers = keepContainers; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchInfo.java new file mode 100644 index 0000000..db988e7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchInfo.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class ContainerLaunchInfo { + Map localResources; + Map environment; + List commands; + Map serviceData; + String tokens; + Map acls; + + public ContainerLaunchInfo() { + localResources = new HashMap(); + environment = new HashMap(); + commands = new ArrayList(); + serviceData = new HashMap(); + tokens = new String(); + acls = new HashMap(); + } + + public Map getLocalResources() { + return localResources; + } + public Map getEnvironment() { + return environment; + } + public List getCommands() { + return commands; + } + + public Map getServiceData() { + return serviceData; + } + + public String getTokens() { + return tokens; + } + + public Map getAcls() { + return acls; + } + public void setLocalResources(Map localResources) { + this.localResources = localResources; + } + public void setEnvironment(Map environment) { + this.environment = environment; + } + public void setCommands(List commands) { + this.commands = commands; + } + + public void setServiceData(Map serviceData) { + this.serviceData = serviceData; + } + + public void setTokens(String tokens) { + this.tokens = tokens; + } + + public void setAcls(Map acls) { + this.acls = acls; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java new file mode 100644 index 0000000..67a85b4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.net.URI; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class LocalResourceInfo { + + URI url; + LocalResourceType type; + LocalResourceVisibility visibility; + long size; + long timestamp; + String pattern; + + public URI getUrl() { + return url; + } + public LocalResourceType getType() { + return type; + } + + public LocalResourceVisibility getVisibility() { + return visibility; + } + + public long getSize() { + return size; + } + + public long getTimestamp() { + return timestamp; + } + public String getPattern() { + return pattern; + } + public void setUrl(URI url) { + this.url = url; + } + public void setType(LocalResourceType type) { + this.type = type; + } + public void setVisibility(LocalResourceVisibility visibility) { + this.visibility = visibility; + } + public void setSize(long size) { + if(size <= 0) { + throw new IllegalArgumentException("size must be greater than 0"); + } + this.size = size; + } + public void setTimestamp(long timestamp) { + if(timestamp <= 0) { + throw new IllegalArgumentException("timestamp must be greater than 0"); + } + this.timestamp = timestamp; + } + public void setPattern(String pattern) { + this.pattern = pattern; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplicationInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplicationInfo.java new file mode 100644 index 0000000..59350b4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplicationInfo.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Resource; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class NewApplicationInfo { + + protected String id; + protected ResourceInfo resource; + + public NewApplicationInfo() { + return; + } + + public NewApplicationInfo(ApplicationId appId, Resource rs) { + id = appId.toString(); + resource = new ResourceInfo(rs); + } + + public String getId() { + return id; + } + + public ResourceInfo getResource() { + return resource; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java index 6b4422c..043351c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java @@ -50,4 +50,12 @@ public int getvCores() { public String toString() { return ""; } + + public void setMemory(int memory) { + this.memory = memory; + } + + public void setvCores(int vCores) { + this.vCores = vCores; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMCustomAuthFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMCustomAuthFilter.java new file mode 100644 index 0000000..d0a0404 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMCustomAuthFilter.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import java.util.Enumeration; +import java.util.Properties; + +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; + +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; +import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler; + +import com.google.inject.Singleton; + +/* + * Helper class to allow testing of RM web services which require authorization + * Add this class as a filter in the Guice injector for the MockRM + * + */ + +@Singleton +public class TestRMCustomAuthFilter extends AuthenticationFilter { + + @Override + protected Properties getConfiguration(String configPrefix, FilterConfig filterConfig) throws ServletException { + Properties props = new Properties(); + Enumeration names = filterConfig.getInitParameterNames(); + while (names.hasMoreElements()) { + String name = (String) names.nextElement(); + if (name.startsWith(configPrefix)) { + String value = filterConfig.getInitParameter(name); + props.put(name.substring(configPrefix.length()), value); + } + } + props.put(AuthenticationFilter.AUTH_TYPE, "simple"); + props.put(PseudoAuthenticationHandler.ANONYMOUS_ALLOWED, "false"); + return props; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 45b3803..91251b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -22,17 +22,27 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import java.io.IOException; import java.io.StringReader; +import java.io.StringWriter; +import java.net.URI; +import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; import javax.ws.rs.core.MediaType; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.Marshaller; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; @@ -47,23 +57,32 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppSubmissionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; +import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; import org.xml.sax.InputSource; +import org.xml.sax.SAXException; import com.google.inject.Guice; import com.google.inject.Injector; import com.google.inject.servlet.GuiceServletContextListener; import com.google.inject.servlet.ServletModule; +import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.ClientResponse.Status; import com.sun.jersey.api.client.UniformInterfaceException; @@ -73,40 +92,83 @@ import com.sun.jersey.test.framework.JerseyTest; import com.sun.jersey.test.framework.WebAppDescriptor; +@RunWith(Parameterized.class) public class TestRMWebServicesApps extends JerseyTest { private static MockRM rm; private static final int CONTAINER_MB = 1024; - - private Injector injector = Guice.createInjector(new ServletModule() { - @Override - protected void configureServlets() { - bind(JAXBContextResolver.class); - bind(RMWebServices.class); - bind(GenericExceptionHandler.class); - Configuration conf = new Configuration(); - conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); - conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, - ResourceScheduler.class); - rm = new MockRM(conf); - bind(ResourceManager.class).toInstance(rm); - bind(RMContext.class).toInstance(rm.getRMContext()); - bind(ApplicationACLsManager.class).toInstance( - rm.getApplicationACLsManager()); - bind(QueueACLsManager.class).toInstance(rm.getQueueACLsManager()); - serve("/*").with(GuiceContainer.class); - } - }); - + + private Injector injector; + private String webserviceUserName = "testuser"; + public class GuiceServletConfig extends GuiceServletContextListener { - + @Override protected Injector getInjector() { return injector; } } + + private Injector getNoAuthInjector() { + return Guice.createInjector(new ServletModule() { + @Override + protected void configureServlets() { + bind(JAXBContextResolver.class); + bind(RMWebServices.class); + bind(GenericExceptionHandler.class); + Configuration conf = new Configuration(); + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, + ResourceScheduler.class); + rm = new MockRM(conf); + bind(ResourceManager.class).toInstance(rm); + bind(RMContext.class).toInstance(rm.getRMContext()); + bind(ApplicationACLsManager.class).toInstance( + rm.getApplicationACLsManager()); + bind(QueueACLsManager.class).toInstance(rm.getQueueACLsManager()); + serve("/*").with(GuiceContainer.class); + } + }); + } + + private Injector getSimpleAuthInjector() { + return Guice.createInjector(new ServletModule() { + @Override + protected void configureServlets() { + bind(JAXBContextResolver.class); + bind(RMWebServices.class); + bind(GenericExceptionHandler.class); + Configuration conf = new Configuration(); + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, + ResourceScheduler.class); + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + // set the admin acls otherwise all users are considered admins + // and we can't test authorization + conf.setStrings(YarnConfiguration.YARN_ADMIN_ACL, "testuser1"); + rm = new MockRM(conf); + bind(ResourceManager.class).toInstance(rm); + bind(RMContext.class).toInstance(rm.getRMContext()); + bind(ApplicationACLsManager.class).toInstance( + rm.getApplicationACLsManager()); + bind(QueueACLsManager.class).toInstance(rm.getQueueACLsManager()); + filter("/*").through(TestRMCustomAuthFilter.class); + serve("/*").with(GuiceContainer.class); + } + }); + } + + @Parameters + public static Collection guiceConfigs() { + return Arrays.asList(new Object[][] { + { 0 }, + { 1 } + }); + } + @Before @Override @@ -114,12 +176,20 @@ public void setUp() throws Exception { super.setUp(); } - public TestRMWebServicesApps() { + public TestRMWebServicesApps(int run) { super(new WebAppDescriptor.Builder( "org.apache.hadoop.yarn.server.resourcemanager.webapp") .contextListenerClass(GuiceServletConfig.class) .filterClass(com.google.inject.servlet.GuiceFilter.class) .contextPath("jersey-guice-filter").servletPath("/").build()); + switch(run) { + case 0: + injector = getNoAuthInjector(); + break; + case 1: + injector = getSimpleAuthInjector(); + break; + } } @Test @@ -158,10 +228,11 @@ public void testAppsXML() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").accept(MediaType.APPLICATION_XML) + + ClientResponse response = this.constructWebResource("apps") + .accept(MediaType.APPLICATION_XML) .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); String xml = response.getEntity(String.class); DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); @@ -185,10 +256,8 @@ public void testAppsXMLMulti() throws JSONException, Exception { rm.submitApp(2048, "testwordcount2", "user1"); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").accept(MediaType.APPLICATION_XML) + ClientResponse response = this.constructWebResource("apps") + .accept(MediaType.APPLICATION_XML) .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); String xml = response.getEntity(String.class); @@ -206,10 +275,9 @@ public void testAppsXMLMulti() throws JSONException, Exception { public void testAppsHelper(String path, RMApp app, String media) throws JSONException, Exception { - WebResource r = resource(); - - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path(path).accept(media).get(ClientResponse.class); + + ClientResponse response = this.constructWebResource(path). + accept(media).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); @@ -227,10 +295,8 @@ public void testAppsQueryState() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps") + + ClientResponse response = this.constructWebResource("apps") .queryParam("state", YarnApplicationState.ACCEPTED.toString()) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); @@ -254,11 +320,10 @@ public void testAppsQueryStates() throws JSONException, Exception { amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); MultivaluedMapImpl params = new MultivaluedMapImpl(); params.add("states", YarnApplicationState.ACCEPTED.toString()); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParams(params) + ClientResponse response = this.constructWebResource("apps") + .queryParams(params) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); @@ -270,12 +335,11 @@ public void testAppsQueryStates() throws JSONException, Exception { assertEquals("state not equal to ACCEPTED", "ACCEPTED", array .getJSONObject(0).getString("state")); - r = resource(); params = new MultivaluedMapImpl(); params.add("states", YarnApplicationState.ACCEPTED.toString()); params.add("states", YarnApplicationState.KILLED.toString()); - response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParams(params) + response = this.constructWebResource("apps") + .queryParams(params) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); json = response.getEntity(JSONObject.class); @@ -303,11 +367,10 @@ public void testAppsQueryStatesComma() throws JSONException, Exception { amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); MultivaluedMapImpl params = new MultivaluedMapImpl(); params.add("states", YarnApplicationState.ACCEPTED.toString()); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParams(params) + ClientResponse response = this.constructWebResource("apps") + .queryParams(params) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); @@ -319,12 +382,11 @@ public void testAppsQueryStatesComma() throws JSONException, Exception { assertEquals("state not equal to ACCEPTED", "ACCEPTED", array .getJSONObject(0).getString("state")); - r = resource(); params = new MultivaluedMapImpl(); params.add("states", YarnApplicationState.ACCEPTED.toString() + "," + YarnApplicationState.KILLED.toString()); - response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParams(params) + response = this.constructWebResource("apps") + .queryParams(params) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); json = response.getEntity(JSONObject.class); @@ -348,10 +410,7 @@ public void testAppsQueryStatesNone() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps") + ClientResponse response = this.constructWebResource("apps") .queryParam("states", YarnApplicationState.RUNNING.toString()) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); @@ -367,12 +426,11 @@ public void testAppsQueryStateNone() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps") + ClientResponse response = this.constructWebResource("apps") .queryParam("state", YarnApplicationState.RUNNING.toString()) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); @@ -386,12 +444,11 @@ public void testAppsQueryStatesInvalid() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); try { - r.path("ws").path("v1").path("cluster").path("apps") - .queryParam("states", "INVALID_test") - .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + this.constructWebResource("apps") + .queryParam("states", "INVALID_test") + .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); fail("should have thrown exception on invalid state query"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); @@ -423,12 +480,11 @@ public void testAppsQueryStateInvalid() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); try { - r.path("ws").path("v1").path("cluster").path("apps") - .queryParam("state", "INVALID_test") - .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + this.constructWebResource("apps") + .queryParam("state", "INVALID_test") + .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); fail("should have thrown exception on invalid state query"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); @@ -460,10 +516,8 @@ public void testAppsQueryFinalStatus() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParam("finalStatus", FinalApplicationStatus.UNDEFINED.toString()) + ClientResponse response = this.constructWebResource("apps") + .queryParam("finalStatus", FinalApplicationStatus.UNDEFINED.toString()) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); @@ -483,10 +537,8 @@ public void testAppsQueryFinalStatusNone() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParam("finalStatus", FinalApplicationStatus.KILLED.toString()) + ClientResponse response = this.constructWebResource("apps") + .queryParam("finalStatus", FinalApplicationStatus.KILLED.toString()) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); @@ -501,12 +553,11 @@ public void testAppsQueryFinalStatusInvalid() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); try { - r.path("ws").path("v1").path("cluster").path("apps") - .queryParam("finalStatus", "INVALID_test") - .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); + this.constructWebResource("apps") + .queryParam("finalStatus", "INVALID_test") + .accept(MediaType.APPLICATION_JSON).get(JSONObject.class); fail("should have thrown exception on invalid state query"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); @@ -541,12 +592,8 @@ public void testAppsQueryUser() throws JSONException, Exception { rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - ClientResponse response = r - .path("ws") - .path("v1") - .path("cluster") - .path("apps") + + ClientResponse response = this.constructWebResource("apps") .queryParam("user", UserGroupInformation.getCurrentUser().getShortUserName()) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); @@ -569,10 +616,8 @@ public void testAppsQueryQueue() throws JSONException, Exception { rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParam("queue", "default") + ClientResponse response = this.constructWebResource("apps") + .queryParam("queue", "default") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); @@ -591,9 +636,8 @@ public void testAppsQueryLimit() throws JSONException, Exception { rm.submitApp(CONTAINER_MB); rm.submitApp(CONTAINER_MB); rm.submitApp(CONTAINER_MB); - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParam("limit", "2") + ClientResponse response = this.constructWebResource("apps") + .queryParam("limit", "2") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); @@ -614,9 +658,8 @@ public void testAppsQueryStartBegin() throws JSONException, Exception { rm.submitApp(CONTAINER_MB); rm.submitApp(CONTAINER_MB); rm.submitApp(CONTAINER_MB); - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParam("startedTimeBegin", String.valueOf(start)) + ClientResponse response = this.constructWebResource("apps") + .queryParam("startedTimeBegin", String.valueOf(start)) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); @@ -637,9 +680,8 @@ public void testAppsQueryStartBeginSome() throws JSONException, Exception { long start = System.currentTimeMillis(); Thread.sleep(1); rm.submitApp(CONTAINER_MB); - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParam("startedTimeBegin", String.valueOf(start)) + ClientResponse response = this.constructWebResource("apps") + .queryParam("startedTimeBegin", String.valueOf(start)) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); @@ -660,9 +702,8 @@ public void testAppsQueryStartEnd() throws JSONException, Exception { rm.submitApp(CONTAINER_MB); rm.submitApp(CONTAINER_MB); rm.submitApp(CONTAINER_MB); - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParam("startedTimeEnd", String.valueOf(end)) + ClientResponse response = this.constructWebResource("apps") + .queryParam("startedTimeEnd", String.valueOf(end)) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); @@ -682,9 +723,8 @@ public void testAppsQueryStartBeginEnd() throws JSONException, Exception { long end = System.currentTimeMillis(); Thread.sleep(1); rm.submitApp(CONTAINER_MB); - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParam("startedTimeBegin", String.valueOf(start)) + ClientResponse response = this.constructWebResource("apps") + .queryParam("startedTimeBegin", String.valueOf(start)) .queryParam("startedTimeEnd", String.valueOf(end)) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); @@ -714,10 +754,8 @@ public void testAppsQueryFinishBegin() throws JSONException, Exception { 1, ContainerState.COMPLETE); rm.submitApp(CONTAINER_MB); rm.submitApp(CONTAINER_MB); - - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParam("finishedTimeBegin", String.valueOf(start)) + ClientResponse response = this.constructWebResource("apps") + .queryParam("finishedTimeBegin", String.valueOf(start)) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); @@ -747,9 +785,8 @@ public void testAppsQueryFinishEnd() throws JSONException, Exception { rm.submitApp(CONTAINER_MB); long end = System.currentTimeMillis(); - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParam("finishedTimeEnd", String.valueOf(end)) + ClientResponse response = this.constructWebResource("apps") + .queryParam("finishedTimeEnd", String.valueOf(end)) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); @@ -781,9 +818,8 @@ public void testAppsQueryFinishBeginEnd() throws JSONException, Exception { rm.submitApp(CONTAINER_MB); long end = System.currentTimeMillis(); - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParam("finishedTimeBegin", String.valueOf(start)) + ClientResponse response = this.constructWebResource("apps") + .queryParam("finishedTimeBegin", String.valueOf(start)) .queryParam("finishedTimeEnd", String.valueOf(end)) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); @@ -816,9 +852,8 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { rm.submitApp(CONTAINER_MB, "", UserGroupInformation.getCurrentUser() .getShortUserName(), null, false, null, 2, null, "NON-YARN"); - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParam("applicationTypes", "MAPREDUCE") + ClientResponse response = this.constructWebResource("apps") + .queryParam("applicationTypes", "MAPREDUCE") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); @@ -830,9 +865,8 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { assertEquals("MAPREDUCE", array.getJSONObject(0).getString("applicationType")); - r = resource(); response = - r.path("ws").path("v1").path("cluster").path("apps") + this.constructWebResource("apps") .queryParam("applicationTypes", "YARN") .queryParam("applicationTypes", "MAPREDUCE") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); @@ -850,9 +884,8 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { && array.getJSONObject(0).getString("applicationType") .equals("MAPREDUCE"))); - r = resource(); response = - r.path("ws").path("v1").path("cluster").path("apps") + this.constructWebResource("apps") .queryParam("applicationTypes", "YARN,NON-YARN") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); @@ -869,9 +902,8 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { && array.getJSONObject(0).getString("applicationType") .equals("NON-YARN"))); - r = resource(); - response = r.path("ws").path("v1").path("cluster") - .path("apps").queryParam("applicationTypes", "") + response = this.constructWebResource("apps") + .queryParam("applicationTypes", "") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); json = response.getEntity(JSONObject.class); @@ -881,9 +913,8 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 3, array.length()); - r = resource(); response = - r.path("ws").path("v1").path("cluster").path("apps") + this.constructWebResource("apps") .queryParam("applicationTypes", "YARN,NON-YARN") .queryParam("applicationTypes", "MAPREDUCE") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); @@ -895,9 +926,8 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 3, array.length()); - r = resource(); response = - r.path("ws").path("v1").path("cluster").path("apps") + this.constructWebResource("apps") .queryParam("applicationTypes", "YARN") .queryParam("applicationTypes", "") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); @@ -911,9 +941,8 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { assertEquals("YARN", array.getJSONObject(0).getString("applicationType")); - r = resource(); response = - r.path("ws").path("v1").path("cluster").path("apps") + this.constructWebResource("apps") .queryParam("applicationTypes", ",,, ,, YARN ,, ,") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); @@ -926,9 +955,8 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { assertEquals("YARN", array.getJSONObject(0).getString("applicationType")); - r = resource(); response = - r.path("ws").path("v1").path("cluster").path("apps") + this.constructWebResource("apps") .queryParam("applicationTypes", ",,, ,, ,, ,") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); @@ -939,9 +967,8 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 3, array.length()); - r = resource(); response = - r.path("ws").path("v1").path("cluster").path("apps") + this.constructWebResource("apps") .queryParam("applicationTypes", "YARN, ,NON-YARN, ,,") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); @@ -958,9 +985,8 @@ public void testAppsQueryAppTypes() throws JSONException, Exception { && array.getJSONObject(0).getString("applicationType") .equals("NON-YARN"))); - r = resource(); response = - r.path("ws").path("v1").path("cluster").path("apps") + this.constructWebResource("apps") .queryParam("applicationTypes", " YARN, , ,,,") .queryParam("applicationTypes", "MAPREDUCE , ,, ,") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); @@ -1004,9 +1030,7 @@ public void testAppStatistics() throws JSONException, Exception { .getShortUserName(), null, false, null, 2, null, "OTHER"); // zero type, zero state - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("appstatistics") + ClientResponse response = this.constructWebResource("appstatistics") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); @@ -1029,9 +1053,7 @@ public void testAppStatistics() throws JSONException, Exception { } // zero type, one state - r = resource(); - response = r.path("ws").path("v1").path("cluster") - .path("appstatistics") + response = this.constructWebResource("appstatistics") .queryParam("states", YarnApplicationState.ACCEPTED.toString()) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); @@ -1046,9 +1068,7 @@ public void testAppStatistics() throws JSONException, Exception { assertEquals("2", statItems.getJSONObject(0).getString("count")); // one type, zero state - r = resource(); - response = r.path("ws").path("v1").path("cluster") - .path("appstatistics") + response = this.constructWebResource("appstatistics") .queryParam("applicationTypes", "MAPREDUCE") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); @@ -1072,9 +1092,7 @@ public void testAppStatistics() throws JSONException, Exception { } // two types, zero state - r = resource(); - response = r.path("ws").path("v1").path("cluster") - .path("appstatistics") + response = this.constructWebResource("appstatistics") .queryParam("applicationTypes", "MAPREDUCE,OTHER") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); @@ -1094,9 +1112,7 @@ public void testAppStatistics() throws JSONException, Exception { "org.apache.hadoop.yarn.webapp.BadRequestException", className); // one type, two states - r = resource(); - response = r.path("ws").path("v1").path("cluster") - .path("appstatistics") + response = this.constructWebResource("appstatistics") .queryParam("states", YarnApplicationState.FINISHED.toString() + "," + YarnApplicationState.ACCEPTED.toString()) .queryParam("applicationTypes", "MAPREDUCE") @@ -1120,9 +1136,8 @@ public void testAppStatistics() throws JSONException, Exception { assertEquals("1", statItem2.getString("count")); // invalid state - r = resource(); - response = r.path("ws").path("v1").path("cluster") - .path("appstatistics").queryParam("states", "wrong_state") + response = this.constructWebResource("appstatistics") + .queryParam("states", "wrong_state") .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); @@ -1182,12 +1197,11 @@ public void testInvalidApp() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); try { - r.path("ws").path("v1").path("cluster").path("apps") - .path("application_invalid_12").accept(MediaType.APPLICATION_JSON) - .get(JSONObject.class); + this.constructWebResource("apps", "application_invalid_12") + .accept(MediaType.APPLICATION_JSON) + .get(JSONObject.class); fail("should have thrown exception on invalid appid"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); @@ -1218,12 +1232,11 @@ public void testNonexistApp() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); try { - r.path("ws").path("v1").path("cluster").path("apps") - .path("application_00000_0099").accept(MediaType.APPLICATION_JSON) - .get(JSONObject.class); + this.constructWebResource("apps", "application_00000_0099") + .accept(MediaType.APPLICATION_JSON) + .get(JSONObject.class); fail("should have thrown exception on invalid appid"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); @@ -1251,9 +1264,8 @@ public void testNonexistApp() throws JSONException, Exception { public void testSingleAppsHelper(String path, RMApp app, String media) throws JSONException, Exception { - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").path(path).accept(media).get(ClientResponse.class); + ClientResponse response = this.constructWebResource("apps", path) + .accept(media).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); @@ -1267,9 +1279,8 @@ public void testSingleAppsXML() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").path(app1.getApplicationId().toString()) + ClientResponse response = this.constructWebResource("apps", + app1.getApplicationId().toString()) .accept(MediaType.APPLICATION_XML).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); String xml = response.getEntity(String.class); @@ -1442,12 +1453,11 @@ public void testInvalidAppAttempts() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); try { - r.path("ws").path("v1").path("cluster").path("apps") - .path("application_invalid_12").accept(MediaType.APPLICATION_JSON) - .get(JSONObject.class); + this.constructWebResource("apps", "application_invalid_12") + .accept(MediaType.APPLICATION_JSON) + .get(JSONObject.class); fail("should have thrown exception on invalid appid"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); @@ -1478,12 +1488,11 @@ public void testNonexistAppAttempts() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); rm.submitApp(CONTAINER_MB, "testwordcount", "user1"); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); try { - r.path("ws").path("v1").path("cluster").path("apps") - .path("application_00000_0099").accept(MediaType.APPLICATION_JSON) - .get(JSONObject.class); + this.constructWebResource("apps", "application_00000_0099") + .accept(MediaType.APPLICATION_JSON) + .get(JSONObject.class); fail("should have thrown exception on invalid appid"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); @@ -1511,9 +1520,8 @@ public void testNonexistAppAttempts() throws JSONException, Exception { public void testAppAttemptsHelper(String path, RMApp app, String media) throws JSONException, Exception { - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").path(path).path("appattempts").accept(media) + ClientResponse response = this.constructWebResource("apps", path, "appattempts") + .accept(media) .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); @@ -1541,10 +1549,9 @@ public void testAppAttemptsXML() throws JSONException, Exception { MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", user); amNodeManager.nodeHeartbeat(true); - WebResource r = resource(); - ClientResponse response = r.path("ws").path("v1").path("cluster") - .path("apps").path(app1.getApplicationId().toString()) - .path("appattempts").accept(MediaType.APPLICATION_XML) + ClientResponse response = this.constructWebResource("apps", + app1.getApplicationId().toString(), "appattempts") + .accept(MediaType.APPLICATION_XML) .get(ClientResponse.class); assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); String xml = response.getEntity(String.class); @@ -1611,6 +1618,416 @@ public void verifyAppAttemptInfoGeneric(RMAppAttempt appAttempt, int id, "logsLink doesn't contain user info", logsLink.endsWith("/" + user)); } - + + private WebResource constructWebResource(WebResource r, String... paths) { + WebResource rt = r; + for(String path: paths) { + rt = rt.path(path); + } + if(rm.getConfig().getBoolean(YarnConfiguration.YARN_ACL_ENABLE, false) == true) { + rt = rt.queryParam("user.name", webserviceUserName); + } + return rt; + } + + private WebResource constructWebResource(String... paths) { + WebResource r = resource(); + WebResource ws = r.path("ws").path("v1").path("cluster"); + return this.constructWebResource(ws, paths); + } + + @Test (timeout = 90000) + public void testSingleAppKill() throws Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + + String[] mediaTypes = {MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}; + for(String mediaType: mediaTypes) { + RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName); + amNodeManager.nodeHeartbeat(true); + ClientResponse response = this.constructWebResource("apps", + app.getApplicationId().toString(), "kill") + .accept(mediaType) + .put(ClientResponse.class); + + if(rm.getConfig().getBoolean(YarnConfiguration.YARN_ACL_ENABLE, false) == false) { + assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); + return; + } + assertEquals(Status.ACCEPTED, response.getClientResponseStatus()); + if(mediaType == MediaType.APPLICATION_JSON) { + verifyAppKillJson(response, RMAppState.ACCEPTED); + } + else { + verifyAppKillXML(response, RMAppState.ACCEPTED); + } + + String locationHeaderValue = response.getHeaders().getFirst("Location"); + Client c = Client.create(); + WebResource tmp = c.resource(locationHeaderValue); + if(rm.getConfig().getBoolean(YarnConfiguration.YARN_ACL_ENABLE, false) == true) { + tmp = tmp.queryParam("user.name", webserviceUserName); + } + response = tmp.get(ClientResponse.class); + assertEquals(Status.OK, response.getClientResponseStatus()); + assertTrue(locationHeaderValue.endsWith("/ws/v1/cluster/apps/" + app.getApplicationId().toString())); + + while(true) { + Thread.sleep(1000); + response = this.constructWebResource("apps", app.getApplicationId().toString(), "kill") + .accept(mediaType) + .put(ClientResponse.class); + assertTrue((response.getClientResponseStatus() == Status.ACCEPTED) || + (response.getClientResponseStatus() == Status.OK)); + if(response.getClientResponseStatus() == Status.OK) { + if(mediaType == MediaType.APPLICATION_JSON) { + verifyAppKillJson(response, RMAppState.KILLED); + } + else { + verifyAppKillXML(response, RMAppState.KILLED); + } + break; + } + } + } + + rm.stop(); + return; + } + + protected static void verifyAppKillJson(ClientResponse response, RMAppState state) throws JSONException { + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 1, json.length()); + JSONObject appObj = json.getJSONObject("app"); + assertEquals("app state incorrect", + state.toString(), appObj.getString("state")); + return; + } + + protected static void verifyAppKillXML(ClientResponse response, RMAppState appState) + throws ParserConfigurationException, IOException, SAXException{ + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("app"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + Element element = (Element) nodes.item(0); + String state = WebServicesTestUtils.getXmlString(element, "state"); + assertEquals("app state incorrect", appState.toString(), state); + return; + } + + @Test (timeout = 30000) + public void testSingleAppKillUnauthorized() throws Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + + String[] mediaTypes = {MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML}; + for(String mediaType: mediaTypes) { + RMApp app = rm.submitApp(CONTAINER_MB, "test", "someuser"); + amNodeManager.nodeHeartbeat(true); + ClientResponse response = this.constructWebResource("apps", + app.getApplicationId().toString(), "kill") + .accept(mediaType) + .put(ClientResponse.class); + if(rm.getConfig().getBoolean(YarnConfiguration.YARN_ACL_ENABLE, false) == false) { + assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); + } + else { + assertEquals(Status.FORBIDDEN, response.getClientResponseStatus()); + } + } + + // there's a condition where if the rm is stopped while an app + // is being killed, it can lead to the test timing out, hence the sleep + Thread.sleep(3000); + + rm.stop(); + return; + + } + + @Test + public void testSingleAppKillInvalidId() throws Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + amNodeManager.nodeHeartbeat(true); + String[] testAppIds = {"application_1391705042196_0001", "random_string"}; + for(String testAppId: testAppIds) { + ClientResponse response = this.constructWebResource("apps", testAppId, "kill") + .accept(MediaType.APPLICATION_XML) + .put(ClientResponse.class); + if(rm.getConfig().getBoolean(YarnConfiguration.YARN_ACL_ENABLE, false) == false) { + assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); + continue; + } + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); + } + rm.stop(); + return; + } + + @Test + public void testGetNewApplicationIdAndSubmit() throws Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + amNodeManager.nodeHeartbeat(true); + String mediaTypes[] = { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }; + String id = ""; + for(int i = 0; i < 2; ++i) { + for(String acceptMedia: mediaTypes) { + for(String contentMedia: mediaTypes) { + ClientResponse response = this.constructWebResource("app", "id") + .accept(acceptMedia) + .post(ClientResponse.class); + if(rm.getConfig().getBoolean(YarnConfiguration.YARN_ACL_ENABLE, false) == false) { + assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); + } + else { + if(acceptMedia.equals(MediaType.APPLICATION_XML)) { + assertEquals(Status.OK, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("newApplicationInfo"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + Element element = (Element) nodes.item(0); + id = WebServicesTestUtils.getXmlString(element, "id"); + assertTrue(ConverterUtils.toApplicationId(id) != null); + } + else { + assertEquals(Status.OK, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 2, json.length()); + id = json.getString("id"); + assertTrue(ConverterUtils.toApplicationId(id) != null); + } + switch(i) { + case 0: + testAppSubmit(id, acceptMedia, contentMedia); + break; + case 1: + testAppSubmitErrors(id, acceptMedia, contentMedia); + break; + } + Thread.sleep(500); + } + } + } + } + rm.stop(); + return; + } + + public void testAppSubmit(String appId, String acceptMedia, String contentMedia) throws Exception { + + AppSubmissionInfo appInfo = new AppSubmissionInfo(); + appInfo.setApplicationId(appId); + appInfo.setApplicationName("test"); + appInfo.setPriority(3); + appInfo.setMaxAppAttempts(2); + appInfo.setQueue("testqueue"); + appInfo.setApplicationType("test-type"); + HashMap lr = new HashMap(); + LocalResourceInfo y = new LocalResourceInfo(); + y.setUrl(new URI("http://www.test.com/file.txt")); + y.setSize(100); + y.setTimestamp(System.currentTimeMillis()); + y.setType(LocalResourceType.FILE); + y.setVisibility(LocalResourceVisibility.APPLICATION); + lr.put("example", y); + appInfo.getContainerLaunchContext().setLocalResources(lr); + appInfo.getResource().setMemory(1024); + appInfo.getResource().setvCores(1); + + ClientResponse response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + assertEquals(Status.ACCEPTED, response.getClientResponseStatus()); + assertTrue(response.getHeaders().getFirst("Location").endsWith("/app/" + appId)); + RMApp app = rm.getRMContext().getRMApps().get(ConverterUtils.toApplicationId(appId)); + assertEquals("test", app.getName()); + assertEquals(webserviceUserName, app.getUser()); + assertEquals(2, app.getMaxAppAttempts()); + assertEquals("testqueue", app.getQueue()); + assertEquals("test-type", app.getApplicationType()); + + response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + return; + } + + public void testAppSubmitErrors(String appId, String acceptMedia, String contentMedia) throws Exception { + + AppSubmissionInfo appInfo = new AppSubmissionInfo(); + ClientResponse response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.setApplicationId(appId); + response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.setApplicationName("test"); + response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.setPriority(3); + appInfo.setMaxAppAttempts(2); + appInfo.setQueue("testqueue"); + appInfo.setApplicationType("test-type"); + + response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + // missing local resource info + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + HashMap lr = new HashMap(); + LocalResourceInfo y = new LocalResourceInfo(); + y.setUrl(new URI("http://www.test.com/file.txt")); + y.setSize(100); + y.setTimestamp(System.currentTimeMillis()); + y.setType(LocalResourceType.FILE); + y.setVisibility(LocalResourceVisibility.APPLICATION); + lr.put("example", y); + appInfo.getContainerLaunchContext().setLocalResources(lr); + response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + // missing resource info + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.getResource().setMemory( + rm.getConfig().getInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB) + 1); + appInfo.getResource().setvCores(1); + response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + // bad resource info + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.getResource().setvCores( + rm.getConfig().getInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES) + 1); + appInfo.getResource().setMemory(CONTAINER_MB); + response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + // bad resource info + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.getResource().setvCores(0); + appInfo.getResource().setMemory(0); + response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + // bad resource info + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + return; + } + + @Test + public void testAppSubmitBadJsonXML() throws Exception { + + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + amNodeManager.nodeHeartbeat(true); + String id = ""; + + AppSubmissionInfo appInfo = new AppSubmissionInfo(); + appInfo.setApplicationName("test"); + appInfo.setPriority(3); + appInfo.setMaxAppAttempts(2); + appInfo.setQueue("testqueue"); + appInfo.setApplicationType("test-type"); + HashMap lr = new HashMap(); + LocalResourceInfo y = new LocalResourceInfo(); + y.setUrl(new URI("http://www.test.com/file.txt")); + y.setSize(100); + y.setTimestamp(System.currentTimeMillis()); + y.setType(LocalResourceType.FILE); + y.setVisibility(LocalResourceVisibility.APPLICATION); + lr.put("example", y); + appInfo.getContainerLaunchContext().setLocalResources(lr); + appInfo.getResource().setMemory(1024); + appInfo.getResource().setvCores(1); + + JAXBContext context = JAXBContext.newInstance(AppSubmissionInfo.class); + Marshaller m = context.createMarshaller(); + + ClientResponse response; + + for(int i = 0; i < 2; ++i) { + response = this.constructWebResource("app", "id") + .accept(MediaType.APPLICATION_XML) + .post(ClientResponse.class); + if(rm.getConfig().getBoolean(YarnConfiguration.YARN_ACL_ENABLE, false) == false) { + assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); + } + else { + assertEquals(Status.OK, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("newApplicationInfo"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + Element element = (Element) nodes.item(0); + id = WebServicesTestUtils.getXmlString(element, "id"); + assertTrue(ConverterUtils.toApplicationId(id) != null); + appInfo.setApplicationId(id); + StringWriter sw = new StringWriter(); + m.marshal(appInfo, sw); + String body = sw.toString(); + switch(i) { + case 0: + body = body.replace("FILE", "FILES12"); + break; + case 1: + body = body.replace("APPLICATION", "APPLICATION@#"); + break; + } + response = this.constructWebResource("app", id) + .accept(MediaType.APPLICATION_XML).entity(body, MediaType.APPLICATION_XML) + .post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + } + } + rm.stop(); + } + + @After + @Override + public void tearDown() throws Exception { + if(rm != null) { + rm.stop(); + } + super.tearDown(); + } }