diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index 529df11..1ee04f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -88,7 +88,7 @@ public static ApplicationSubmissionContext newInstance( int maxAppAttempts, Resource resource, String applicationType) { return newInstance(applicationId, applicationName, queue, priority, amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts, - resource, null, false); + resource, applicationType, false); } @Public diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java index 74180f3..1e53e02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java @@ -21,10 +21,12 @@ import java.io.IOException; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; import javax.ws.rs.ext.ExceptionMapper; import javax.ws.rs.ext.Provider; +import javax.xml.bind.UnmarshalException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -87,6 +89,9 @@ public Response toResponse(Exception e) { s = Response.Status.BAD_REQUEST; } else if (e instanceof BadRequestException) { s = Response.Status.BAD_REQUEST; + } else if (e instanceof WebApplicationException + && e.getCause() instanceof UnmarshalException) { + s = Response.Status.BAD_REQUEST; } else { LOG.warn("INTERNAL_SERVER_ERROR", e); s = Response.Status.INTERNAL_SERVER_ERROR; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 52cac3d..bb20b2a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import java.io.IOException; +import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Collection; @@ -34,6 +35,7 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.Consumes; import javax.ws.rs.GET; +import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -50,16 +52,26 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -80,6 +92,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppSubmissionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; @@ -87,6 +100,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplicationInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; @@ -100,6 +115,7 @@ import com.google.inject.Inject; import com.google.inject.Singleton; +import com.sun.jersey.core.util.Base64; @Singleton @Path("/ws/v1/cluster") @@ -748,5 +764,202 @@ protected Boolean hasQueueAcess(RMApp app, } return true; } + + //reuse the code in ClientRMService to create new app + @POST + @Path("/app/id") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public NewApplicationInfo createNewApplication( + @Context HttpServletRequest hsr) + throws AuthorizationException{ + + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr); + if(callerUGI == null) { + throw new AuthorizationException("Unable to obtain user " + + "name, user not authenticated"); + } + GetNewApplicationRequest req = recordFactory. + newRecordInstance(GetNewApplicationRequest.class); + GetNewApplicationResponse resp; + try { + resp = rm.getClientRMService().getNewApplication(req); + } + catch(YarnException e) { + String msg = "Unable to create new app from RM web service"; + LOG.error(msg, e); + throw new YarnRuntimeException(msg, e); + } + return new NewApplicationInfo(resp.getApplicationId(), + resp.getMaximumResourceCapability()); + } + + @POST + @Path("/app/{appid}") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response submitNewApplication(AppSubmissionInfo newApp, + @Context HttpServletRequest hsr, + @PathParam("appid") String appId) + throws AuthorizationException, IOException, InterruptedException{ + + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr); + if(callerUGI == null) { + throw new AuthorizationException("Unable to obtain user name, " + + "user not authenticated"); + } + + RMApp app = null; + try { + app = getRMAppForAppId(appId); + } + catch(NotFoundException e) { + // do nothing this is the expected response + ; + } + if(app != null) { + throw new BadRequestException("App with appId '" + + appId + "' already exists"); + } + + if(appId.equals(newApp.getApplicationId()) == false) { + throw new BadRequestException("appid in url and submitted " + + "info don't match"); + } + if(newApp.getApplicationName().isEmpty()) { + throw new BadRequestException("Application name missing"); + } + if(newApp.getResource().getMemory() == 0 || + newApp.getResource().getvCores() == 0) { + throw new BadRequestException("Requested resource memory " + + "and/or cores are 0"); + } + + ApplicationSubmissionContext appContext = createAppSubmissionContext(newApp); + final SubmitApplicationRequest req = + SubmitApplicationRequest.newInstance(appContext); + + callerUGI.doAs(new + PrivilegedExceptionAction() { + @Override + public SubmitApplicationResponse run() throws IOException, YarnException { + return rm.getClientRMService().submitApplication(req); + } + }); + StringBuffer url = hsr.getRequestURL(); + return Response.status(Status.ACCEPTED) + .header("Location", url).build(); + } + + protected ApplicationSubmissionContext + createAppSubmissionContext(AppSubmissionInfo newApp) + throws BadRequestException { + + // create local resources and app submission context + + ApplicationId appid = ConverterUtils + .toApplicationId(recordFactory, newApp.getApplicationId()); + ApplicationSubmissionContext appContext = ApplicationSubmissionContext + .newInstance( + appid, + newApp.getApplicationName(), + newApp.getQueue(), + Priority.newInstance(newApp.getPriority()), + createContainerLaunchContext(newApp), + newApp.isUnmanagedAM(), + newApp.isCancelTokensWhenComplete(), + newApp.getMaxAppAttempts(), + createAppSubmissionContextResource(newApp), + newApp.getApplicationType()); + + appContext.setKeepContainersAcrossApplicationAttempts( + newApp.isKeepContainers()); + + return appContext; + } + + protected Resource + createAppSubmissionContextResource(AppSubmissionInfo newApp) + throws BadRequestException { + if(newApp.getResource().getvCores() > + rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)) { + String msg = "Requested more cores than configured max"; + throw new BadRequestException(msg); + } + if(newApp.getResource().getMemory() > + rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)) { + String msg = "Requested more memory than configured max"; + throw new BadRequestException(msg); + } + Resource r = Resource.newInstance( + newApp.getResource().getMemory(), + newApp.getResource().getvCores()); + return r; + } + + protected ContainerLaunchContext + createContainerLaunchContext(AppSubmissionInfo newApp) + throws BadRequestException { + + // create container launch context + + HashMap hmap = new HashMap(); + for(Map.Entry entry: + newApp.getContainerLaunchContext().getServiceData().entrySet()) { + if(entry.getValue().isEmpty() == false) { + hmap.put(entry.getKey(), ByteBuffer.wrap( + Base64.decode(entry.getValue()))); + } + } + + // verify local resource values + if(newApp.getContainerLaunchContext().getLocalResources().size() <= 0) { + throw new BadRequestException("Request must specify at " + + "least one local resource"); + } + HashMap hlr = new HashMap(); + for(Map.Entry entry: + newApp.getContainerLaunchContext().getLocalResources().entrySet()) { + LocalResourceInfo l = entry.getValue(); + if(l.getUrl() == null) { + throw new BadRequestException("File URI not set or was set " + + "to an incorrect " + + "value in the LocalResource for '" + entry.getKey() + "'"); + } + if(l.getType() == null) { + throw new BadRequestException("File type not set or was set" + + " to an incorrect " + + "value in the LocalResource for '" + entry.getKey() + "'"); + } + if(l.getVisibility() == null) { + throw new BadRequestException("File visibility not set or " + + "was set to an incorrect " + + "value in the LocalResource for '" + entry.getKey() + "'"); + } + LocalResource lr = LocalResource.newInstance( + ConverterUtils.getYarnUrlFromURI(l.getUrl()), + l.getType(), l.getVisibility(), l.getSize(), l.getTimestamp()); + hlr.put(entry.getKey(), lr); + } + + ContainerLaunchContext ctx = ContainerLaunchContext.newInstance( + hlr, + newApp.getContainerLaunchContext().getEnvironment(), + newApp.getContainerLaunchContext().getCommands(), + hmap, + null, + newApp.getContainerLaunchContext().getAcls()); + + if(newApp.getContainerLaunchContext().getTokens().isEmpty() == false) { + ctx.getTokens().put( + Base64.decode(newApp.getContainerLaunchContext().getTokens())); + } + return ctx; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppSubmissionInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppSubmissionInfo.java new file mode 100644 index 0000000..9bca076 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppSubmissionInfo.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.Priority; + +@XmlRootElement(name="appsubmission") +@XmlAccessorType(XmlAccessType.FIELD) +public class AppSubmissionInfo { + + String applicationId; + String applicationName; + String queue; + int priority; + ContainerLaunchInfo containerInfo; + boolean isUnmanagedAM; + boolean cancelTokensWhenComplete; + int maxAppAttempts; + ResourceInfo resource; + String applicationType; + boolean keepContainers; + + public AppSubmissionInfo() { + applicationId = new String(); + applicationName = new String(); + containerInfo = new ContainerLaunchInfo(); + resource = new ResourceInfo(); + priority = Priority.UNDEFINED.getPriority(); + isUnmanagedAM = false; + } + + public String getApplicationId() { + return applicationId; + } + + public String getApplicationName() { + return applicationName; + } + public String getQueue() { + return queue; + } + public int getPriority() { + return priority; + } + public ContainerLaunchInfo getContainerLaunchContext() { + return containerInfo; + } + public boolean isUnmanagedAM() { + return isUnmanagedAM; + } + public boolean isCancelTokensWhenComplete() { + return cancelTokensWhenComplete; + } + public int getMaxAppAttempts() { + return maxAppAttempts; + } + public ResourceInfo getResource() { + return resource; + } + public String getApplicationType() { + return applicationType; + } + public boolean isKeepContainers() { + return keepContainers; + } + public void setApplicationId(String applicationId) { + this.applicationId = applicationId; + } + public void setApplicationName(String applicationName) { + this.applicationName = applicationName; + } + public void setQueue(String queue) { + this.queue = queue; + } + public void setPriority(int priority) { + this.priority = priority; + } + public void setContainerLaunchContext(ContainerLaunchInfo containerLaunchContext) { + this.containerInfo = containerLaunchContext; + } + public void setUnmanagedAM(boolean isUnmanagedAM) { + this.isUnmanagedAM = isUnmanagedAM; + } + public void setCancelTokensWhenComplete(boolean cancelTokensWhenComplete) { + this.cancelTokensWhenComplete = cancelTokensWhenComplete; + } + public void setMaxAppAttempts(int maxAppAttempts) { + this.maxAppAttempts = maxAppAttempts; + } + public void setResource(ResourceInfo resource) { + this.resource = resource; + } + public void setApplicationType(String applicationType) { + this.applicationType = applicationType; + } + public void setKeepContainers(boolean keepContainers) { + this.keepContainers = keepContainers; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchInfo.java new file mode 100644 index 0000000..24a24d7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchInfo.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; + +@XmlRootElement(name="containerinfo") +@XmlAccessorType(XmlAccessType.FIELD) +public class ContainerLaunchInfo { + Map localResources; + Map environment; + List commands; + Map serviceData; + String tokens; + Map acls; + + public ContainerLaunchInfo() { + localResources = new HashMap(); + environment = new HashMap(); + commands = new ArrayList(); + serviceData = new HashMap(); + tokens = new String(); + acls = new HashMap(); + } + + public Map getLocalResources() { + return localResources; + } + public Map getEnvironment() { + return environment; + } + public List getCommands() { + return commands; + } + + public Map getServiceData() { + return serviceData; + } + + public String getTokens() { + return tokens; + } + + public Map getAcls() { + return acls; + } + public void setLocalResources(Map localResources) { + this.localResources = localResources; + } + public void setEnvironment(Map environment) { + this.environment = environment; + } + public void setCommands(List commands) { + this.commands = commands; + } + + public void setServiceData(Map serviceData) { + this.serviceData = serviceData; + } + + public void setTokens(String tokens) { + this.tokens = tokens; + } + + public void setAcls(Map acls) { + this.acls = acls; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java new file mode 100644 index 0000000..b282000 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.net.URI; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; + +@XmlRootElement(name="localresource") +@XmlAccessorType(XmlAccessType.FIELD) +public class LocalResourceInfo { + + URI url; + LocalResourceType type; + LocalResourceVisibility visibility; + long size; + long timestamp; + String pattern; + + public URI getUrl() { + return url; + } + public LocalResourceType getType() { + return type; + } + + public LocalResourceVisibility getVisibility() { + return visibility; + } + + public long getSize() { + return size; + } + + public long getTimestamp() { + return timestamp; + } + public String getPattern() { + return pattern; + } + public void setUrl(URI url) { + this.url = url; + } + public void setType(LocalResourceType type) { + this.type = type; + } + public void setVisibility(LocalResourceVisibility visibility) { + this.visibility = visibility; + } + public void setSize(long size) { + if(size <= 0) { + throw new IllegalArgumentException("size must be greater than 0"); + } + this.size = size; + } + public void setTimestamp(long timestamp) { + if(timestamp <= 0) { + throw new IllegalArgumentException("timestamp must be greater than 0"); + } + this.timestamp = timestamp; + } + public void setPattern(String pattern) { + this.pattern = pattern; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplicationInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplicationInfo.java new file mode 100644 index 0000000..f02b669 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplicationInfo.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Resource; + +@XmlRootElement(name="newapp") +@XmlAccessorType(XmlAccessType.FIELD) +public class NewApplicationInfo { + + protected String id; + protected ResourceInfo resource; + + public NewApplicationInfo() { + return; + } + + public NewApplicationInfo(ApplicationId appId, Resource rs) { + id = appId.toString(); + resource = new ResourceInfo(rs); + } + + public String getId() { + return id; + } + + public ResourceInfo getResource() { + return resource; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java index 6b4422c..043351c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java @@ -50,4 +50,12 @@ public int getvCores() { public String toString() { return ""; } + + public void setMemory(int memory) { + this.memory = memory; + } + + public void setvCores(int vCores) { + this.vCores = vCores; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index b5cc77c..3a82ca2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -25,8 +25,10 @@ import java.io.IOException; import java.io.StringReader; import java.io.StringWriter; +import java.net.URI; import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; import javax.ws.rs.core.MediaType; import javax.xml.bind.JAXBContext; @@ -39,6 +41,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; @@ -54,7 +58,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppSubmissionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; import org.codehaus.jettison.json.JSONArray; @@ -1630,8 +1637,7 @@ private WebResource constructWebResource(WebResource r, String... paths) { for (String path : paths) { rt = rt.path(path); } - if (rm.getConfig() - .getBoolean(YarnConfiguration.YARN_ACL_ENABLE, false) == true) { + if (rm.getConfig().getBoolean(YarnConfiguration.YARN_ACL_ENABLE, false) == true) { rt = rt.queryParam("user.name", webserviceUserName); } return rt; @@ -1813,9 +1819,9 @@ public void testSingleAppKillInvalidId() throws Exception { "random_string" }; for (String testAppId : testAppIds) { AppInfo info = new AppInfo(); - ClientResponse response = - this.constructWebResource("apps", testAppId) - .accept(MediaType.APPLICATION_XML).entity(info, MediaType.APPLICATION_XML) + ClientResponse response = this.constructWebResource("apps", testAppId) + .accept(MediaType.APPLICATION_XML) + .entity(info, MediaType.APPLICATION_XML) .put(ClientResponse.class); if (rm.getConfig() .getBoolean(YarnConfiguration.YARN_ACL_ENABLE, false) == false) { @@ -1836,4 +1842,274 @@ public void tearDown() throws Exception { } super.tearDown(); } -} + + @Test + public void testGetNewApplicationIdAndSubmit() throws Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + amNodeManager.nodeHeartbeat(true); + String mediaTypes[] = + { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }; + String id = ""; + for (int i = 0; i < 2; ++i) { + for (String acceptMedia : mediaTypes) { + for (String contentMedia : mediaTypes) { + ClientResponse response = + this.constructWebResource("app", "id").accept(acceptMedia) + .post(ClientResponse.class); + if (rm.getConfig() + .getBoolean(YarnConfiguration.YARN_ACL_ENABLE, false) == false) { + assertEquals(Status.UNAUTHORIZED, + response.getClientResponseStatus()); + } else { + if (acceptMedia.equals(MediaType.APPLICATION_XML)) { + assertEquals(Status.OK, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = + DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("newapp"); + assertEquals("incorrect number of elements", + 1, nodes.getLength()); + Element element = (Element) nodes.item(0); + id = WebServicesTestUtils.getXmlString(element, "id"); + assertTrue(ConverterUtils.toApplicationId(id) != null); + } else { + assertEquals(Status.OK, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, + response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 2, json.length()); + id = json.getString("id"); + assertTrue(ConverterUtils.toApplicationId(id) != null); + } + switch (i) { + case 0: + testAppSubmit(id, acceptMedia, contentMedia); + break; + case 1: + testAppSubmitErrors(id, acceptMedia, contentMedia); + break; + } + Thread.sleep(500); + } + } + } + } + rm.stop(); + return; + } + + public void testAppSubmit(String appId, String acceptMedia, + String contentMedia) throws Exception { + + AppSubmissionInfo appInfo = new AppSubmissionInfo(); + appInfo.setApplicationId(appId); + appInfo.setApplicationName("test"); + appInfo.setPriority(3); + appInfo.setMaxAppAttempts(2); + appInfo.setQueue("testqueue"); + appInfo.setApplicationType("test-type"); + HashMap lr = + new HashMap(); + LocalResourceInfo y = new LocalResourceInfo(); + y.setUrl(new URI("http://www.test.com/file.txt")); + y.setSize(100); + y.setTimestamp(System.currentTimeMillis()); + y.setType(LocalResourceType.FILE); + y.setVisibility(LocalResourceVisibility.APPLICATION); + lr.put("example", y); + appInfo.getContainerLaunchContext().setLocalResources(lr); + appInfo.getResource().setMemory(1024); + appInfo.getResource().setvCores(1); + + ClientResponse response = + this.constructWebResource("app", appId).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + assertEquals(Status.ACCEPTED, response.getClientResponseStatus()); + assertTrue(response.getHeaders().getFirst("Location") + .endsWith("/app/" + appId)); + RMApp app = rm.getRMContext().getRMApps() + .get(ConverterUtils.toApplicationId(appId)); + assertEquals("test", app.getName()); + assertEquals(webserviceUserName, app.getUser()); + assertEquals(2, app.getMaxAppAttempts()); + assertEquals("testqueue", app.getQueue()); + assertEquals("test-type", app.getApplicationType()); + + response = this.constructWebResource("app", appId).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + return; + } + + public void testAppSubmitErrors(String appId, String acceptMedia, + String contentMedia) throws Exception { + + AppSubmissionInfo appInfo = new AppSubmissionInfo(); + ClientResponse response = + this.constructWebResource("app", appId).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.setApplicationId(appId); + response = + this.constructWebResource("app", appId).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.setApplicationName("test"); + response = + this.constructWebResource("app", appId).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.setPriority(3); + appInfo.setMaxAppAttempts(2); + appInfo.setQueue("testqueue"); + appInfo.setApplicationType("test-type"); + + response = + this.constructWebResource("app", appId).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + // missing local resource info + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + HashMap lr = + new HashMap(); + LocalResourceInfo y = new LocalResourceInfo(); + y.setUrl(new URI("http://www.test.com/file.txt")); + y.setSize(100); + y.setTimestamp(System.currentTimeMillis()); + y.setType(LocalResourceType.FILE); + y.setVisibility(LocalResourceVisibility.APPLICATION); + lr.put("example", y); + appInfo.getContainerLaunchContext().setLocalResources(lr); + response = + this.constructWebResource("app", appId).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + // missing resource info + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.getResource().setMemory( + rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB) + 1); + appInfo.getResource().setvCores(1); + response = + this.constructWebResource("app", appId).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + // bad resource info + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.getResource().setvCores( + rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES) + 1); + appInfo.getResource().setMemory(CONTAINER_MB); + response = + this.constructWebResource("app", appId).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + // bad resource info + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.getResource().setvCores(0); + appInfo.getResource().setMemory(0); + response = + this.constructWebResource("app", appId).accept(acceptMedia) + .entity(appInfo, contentMedia).post(ClientResponse.class); + // bad resource info + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + return; + } + + @Test + public void testAppSubmitBadJsonXML() throws Exception { + + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + amNodeManager.nodeHeartbeat(true); + String id = ""; + + AppSubmissionInfo appInfo = new AppSubmissionInfo(); + appInfo.setApplicationName("test"); + appInfo.setPriority(3); + appInfo.setMaxAppAttempts(2); + appInfo.setQueue("testqueue"); + appInfo.setApplicationType("test-type"); + HashMap lr = + new HashMap(); + LocalResourceInfo y = new LocalResourceInfo(); + y.setUrl(new URI("http://www.test.com/file.txt")); + y.setSize(100); + y.setTimestamp(System.currentTimeMillis()); + y.setType(LocalResourceType.FILE); + y.setVisibility(LocalResourceVisibility.APPLICATION); + lr.put("example", y); + appInfo.getContainerLaunchContext().setLocalResources(lr); + appInfo.getResource().setMemory(1024); + appInfo.getResource().setvCores(1); + + JAXBContext context = JAXBContext.newInstance(AppSubmissionInfo.class); + Marshaller m = context.createMarshaller(); + + ClientResponse response = this.constructWebResource("app", "id") + .accept(MediaType.APPLICATION_XML).post(ClientResponse.class); + if (rm.getConfig() + .getBoolean(YarnConfiguration.YARN_ACL_ENABLE, false) == false) { + assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); + return; + } + assertEquals(Status.OK, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("newapp"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + Element element = (Element) nodes.item(0); + id = WebServicesTestUtils.getXmlString(element, "id"); + assertTrue(ConverterUtils.toApplicationId(id) != null); + appInfo.setApplicationId(id); + StringWriter sw = new StringWriter(); + m.marshal(appInfo, sw); + String body = sw.toString(); + for (int i = 0; i < 3; ++i) { + switch (i) { + case 0: + body = body.replace("FILE", "FILES12"); + break; + case 1: + body = body.replace("APPLICATION", "APPLICATION@#"); + break; + case 2: + body = ""; + break; + } + response = + this.constructWebResource("app", id) + .accept(MediaType.APPLICATION_XML) + .entity(body, MediaType.APPLICATION_XML) + .post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + } + body = "{\"a\" : \"b\"}"; + response = + this.constructWebResource("app", id) + .accept(MediaType.APPLICATION_XML) + .entity(body, MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + rm.stop(); + } +} \ No newline at end of file