diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 5aac857..39dcac4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; import java.io.IOException; +import java.nio.ByteBuffer; import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.Collection; @@ -32,7 +33,9 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.Consumes; import javax.ws.rs.GET; +import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -49,16 +52,26 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -79,6 +92,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppSubmissionInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; @@ -86,6 +100,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NewApplicationInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; @@ -93,12 +109,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.webapp.BadRequestException; import org.apache.hadoop.yarn.webapp.NotFoundException; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import com.google.inject.Inject; import com.google.inject.Singleton; +import com.sun.jersey.core.util.Base64; @Singleton @Path("/ws/v1/cluster") @@ -715,5 +733,168 @@ protected Boolean hasQueueAcess(RMApp app, UserGroupInformation callerUGI, Queue } return true; } + + //reuse the code in ClientRMService to create new app + @POST + @Path("/app/id") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public NewApplicationInfo createNewApplication(@Context HttpServletRequest hsr) + throws AuthorizationException{ + + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr); + if(callerUGI == null) { + throw new AuthorizationException("Unable to obtain user name, user not authenticated"); + } + GetNewApplicationRequest req = recordFactory. + newRecordInstance(GetNewApplicationRequest.class); + GetNewApplicationResponse resp; + try { + resp = rm.getClientRMService().getNewApplication(req); + } + catch(YarnException e) { + String msg = "Unable to create new app from RM web service"; + LOG.error(msg, e); + throw new YarnRuntimeException(msg, e); + } + return new NewApplicationInfo(resp.getApplicationId(), resp.getMaximumResourceCapability()); + } + + @POST + @Path("/app/{appid}") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public Response submitNewApplication(AppSubmissionInfo newApp, @Context HttpServletRequest hsr, + @PathParam("appid") String appId) throws AuthorizationException, IOException, InterruptedException{ + + init(); + UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr); + if(callerUGI == null) { + throw new AuthorizationException("Unable to obtain user name, user not authenticated"); + } + + RMApp app = null; + try { + app = appIdToRMApp(appId); + } + catch(NotFoundException e) { + // do nothing this is the expected response + ; + } + if(app != null) { + throw new BadRequestException("App with appId '" + appId + "' already exists"); + } + + if(appId.equals(newApp.getApplicationId()) == false) { + throw new BadRequestException("appid in url and submitted info don't match"); + } + if(newApp.getApplicationName().isEmpty()) { + throw new BadRequestException("Application name missing"); + } + if(newApp.getResource().getMemory() == 0 || newApp.getResource().getvCores() == 0) { + throw new BadRequestException("Requested resource memory and/or cores are 0"); + } + + ApplicationSubmissionContext appContext = createAppSubmissionContext(newApp); + final SubmitApplicationRequest req = recordFactory.newRecordInstance(SubmitApplicationRequest.class); + req.setApplicationSubmissionContext(appContext); + + callerUGI.doAs(new + PrivilegedExceptionAction() { + @Override + public SubmitApplicationResponse run() throws IOException, YarnException { + return rm.getClientRMService().submitApplication(req); + } + }); + + StringBuffer url = hsr.getRequestURL(); + return Response.status(Status.ACCEPTED) + .header("Location", url).build(); + } + + protected ApplicationSubmissionContext createAppSubmissionContext(AppSubmissionInfo newApp) throws BadRequestException { + + // create local resources and app submission context + ApplicationSubmissionContext appContext = + recordFactory.newRecordInstance(ApplicationSubmissionContext.class); + appContext.setApplicationId(ConverterUtils.toApplicationId(recordFactory, newApp.getApplicationId())); + appContext.setApplicationName(newApp.getApplicationName()); + appContext.setQueue(newApp.getQueue()); + Priority priority = Records.newRecord(Priority.class); + priority.setPriority(newApp.getPriority()); + appContext.setPriority(priority); + appContext.setUnmanagedAM(newApp.isUnmanagedAM()); + appContext.setCancelTokensWhenComplete(newApp.isCancelTokensWhenComplete()); + appContext.setMaxAppAttempts(newApp.getMaxAppAttempts()); + appContext.setApplicationType(newApp.getApplicationType()); + appContext.setKeepContainersAcrossApplicationAttempts(newApp.isKeepContainers()); + appContext.setResource(createAppSubmissionContextResource(newApp)); + appContext.setAMContainerSpec(createContainerLaunchContext(newApp)); + return appContext; + } + + protected Resource createAppSubmissionContextResource(AppSubmissionInfo newApp) throws BadRequestException { + if(newApp.getResource().getvCores() > + rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)) { + throw new BadRequestException("Requested more cores than configured max"); + } + if(newApp.getResource().getMemory() > + rm.getConfig().getInt( + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB)) { + throw new BadRequestException("Requested more memory than configured max"); + } + Resource r = recordFactory.newRecordInstance(Resource.class); + r.setMemory(newApp.getResource().getMemory()); + r.setVirtualCores(newApp.getResource().getvCores()); + return r; + } + + protected ContainerLaunchContext createContainerLaunchContext(AppSubmissionInfo newApp) throws BadRequestException { + + // create container launch context + ContainerLaunchContext ctx = recordFactory.newRecordInstance(ContainerLaunchContext.class); + ctx.setCommands(newApp.getContainerLaunchContext().getCommands()); + ctx.setApplicationACLs(newApp.getContainerLaunchContext().getAcls()); + ctx.setEnvironment(newApp.getContainerLaunchContext().getEnvironment()); + if(newApp.getContainerLaunchContext().getTokens().isEmpty() == false) { + ctx.getTokens().put(Base64.decode(newApp.getContainerLaunchContext().getTokens())); + } + HashMap hmap = new HashMap(); + for(Map.Entry entry: newApp.getContainerLaunchContext().getServiceData().entrySet()) { + if(entry.getValue().isEmpty() == false) { + hmap.put(entry.getKey(), ByteBuffer.wrap(Base64.decode(entry.getValue()))); + } + } + ctx.setServiceData(hmap); + // verify local resource values + if(newApp.getContainerLaunchContext().getLocalResources().size() <= 0) { + throw new BadRequestException("Request must specify at least one local resource"); + } + HashMap hlr = new HashMap(); + for(Map.Entry entry: newApp.getContainerLaunchContext().getLocalResources().entrySet()) { + LocalResourceInfo l = entry.getValue(); + if(l.getUrl() == null) { + throw new BadRequestException("File URI not set or was set to an incorrect " + + "value in the LocalResource for '" + entry.getKey() + "'"); + } + if(l.getType() == null) { + throw new BadRequestException("File type not set or was set to an incorrect " + + "value in the LocalResource for '" + entry.getKey() + "'"); + } + if(l.getVisibility() == null) { + throw new BadRequestException("File visibility not set or was set to an incorrect " + + "value in the LocalResource for '" + entry.getKey() + "'"); + } + LocalResource lr = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(l.getUrl()), + l.getType(), l.getVisibility(), l.getSize(), l.getTimestamp()); + hlr.put(entry.getKey(), lr); + } + ctx.setLocalResources(hlr); + ctx.setApplicationACLs(newApp.getContainerLaunchContext().getAcls()); + return ctx; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppSubmissionInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppSubmissionInfo.java new file mode 100644 index 0000000..3150baf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppSubmissionInfo.java @@ -0,0 +1,120 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.Priority; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class AppSubmissionInfo { + + String applicationId; + String applicationName; + String queue; + int priority; + ContainerLaunchInfo containerLaunchContext; + boolean isUnmanagedAM; + boolean cancelTokensWhenComplete; + int maxAppAttempts; + ResourceInfo resource; + String applicationType; + boolean keepContainers; + + public AppSubmissionInfo() { + applicationId = new String(); + applicationName = new String(); + containerLaunchContext = new ContainerLaunchInfo(); + resource = new ResourceInfo(); + priority = Priority.UNDEFINED.getPriority(); + isUnmanagedAM = false; + } + + public String getApplicationId() { + return applicationId; + } + + public String getApplicationName() { + return applicationName; + } + public String getQueue() { + return queue; + } + public int getPriority() { + return priority; + } + public ContainerLaunchInfo getContainerLaunchContext() { + return containerLaunchContext; + } + public boolean isUnmanagedAM() { + return isUnmanagedAM; + } + public boolean isCancelTokensWhenComplete() { + return cancelTokensWhenComplete; + } + public int getMaxAppAttempts() { + return maxAppAttempts; + } + public ResourceInfo getResource() { + return resource; + } + public String getApplicationType() { + return applicationType; + } + public boolean isKeepContainers() { + return keepContainers; + } + public void setApplicationId(String applicationId) { + this.applicationId = applicationId; + } + public void setApplicationName(String applicationName) { + this.applicationName = applicationName; + } + public void setQueue(String queue) { + this.queue = queue; + } + public void setPriority(int priority) { + this.priority = priority; + } + public void setContainerLaunchContext(ContainerLaunchInfo containerLaunchContext) { + this.containerLaunchContext = containerLaunchContext; + } + public void setUnmanagedAM(boolean isUnmanagedAM) { + this.isUnmanagedAM = isUnmanagedAM; + } + public void setCancelTokensWhenComplete(boolean cancelTokensWhenComplete) { + this.cancelTokensWhenComplete = cancelTokensWhenComplete; + } + public void setMaxAppAttempts(int maxAppAttempts) { + this.maxAppAttempts = maxAppAttempts; + } + public void setResource(ResourceInfo resource) { + this.resource = resource; + } + public void setApplicationType(String applicationType) { + this.applicationType = applicationType; + } + public void setKeepContainers(boolean keepContainers) { + this.keepContainers = keepContainers; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchInfo.java new file mode 100644 index 0000000..db988e7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ContainerLaunchInfo.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.ApplicationAccessType; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class ContainerLaunchInfo { + Map localResources; + Map environment; + List commands; + Map serviceData; + String tokens; + Map acls; + + public ContainerLaunchInfo() { + localResources = new HashMap(); + environment = new HashMap(); + commands = new ArrayList(); + serviceData = new HashMap(); + tokens = new String(); + acls = new HashMap(); + } + + public Map getLocalResources() { + return localResources; + } + public Map getEnvironment() { + return environment; + } + public List getCommands() { + return commands; + } + + public Map getServiceData() { + return serviceData; + } + + public String getTokens() { + return tokens; + } + + public Map getAcls() { + return acls; + } + public void setLocalResources(Map localResources) { + this.localResources = localResources; + } + public void setEnvironment(Map environment) { + this.environment = environment; + } + public void setCommands(List commands) { + this.commands = commands; + } + + public void setServiceData(Map serviceData) { + this.serviceData = serviceData; + } + + public void setTokens(String tokens) { + this.tokens = tokens; + } + + public void setAcls(Map acls) { + this.acls = acls; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java new file mode 100644 index 0000000..67a85b4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/LocalResourceInfo.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.net.URI; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class LocalResourceInfo { + + URI url; + LocalResourceType type; + LocalResourceVisibility visibility; + long size; + long timestamp; + String pattern; + + public URI getUrl() { + return url; + } + public LocalResourceType getType() { + return type; + } + + public LocalResourceVisibility getVisibility() { + return visibility; + } + + public long getSize() { + return size; + } + + public long getTimestamp() { + return timestamp; + } + public String getPattern() { + return pattern; + } + public void setUrl(URI url) { + this.url = url; + } + public void setType(LocalResourceType type) { + this.type = type; + } + public void setVisibility(LocalResourceVisibility visibility) { + this.visibility = visibility; + } + public void setSize(long size) { + if(size <= 0) { + throw new IllegalArgumentException("size must be greater than 0"); + } + this.size = size; + } + public void setTimestamp(long timestamp) { + if(timestamp <= 0) { + throw new IllegalArgumentException("timestamp must be greater than 0"); + } + this.timestamp = timestamp; + } + public void setPattern(String pattern) { + this.pattern = pattern; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplicationInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplicationInfo.java new file mode 100644 index 0000000..59350b4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NewApplicationInfo.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.Resource; + +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class NewApplicationInfo { + + protected String id; + protected ResourceInfo resource; + + public NewApplicationInfo() { + return; + } + + public NewApplicationInfo(ApplicationId appId, Resource rs) { + id = appId.toString(); + resource = new ResourceInfo(rs); + } + + public String getId() { + return id; + } + + public ResourceInfo getResource() { + return resource; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java index 6b4422c..043351c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java @@ -50,4 +50,12 @@ public int getvCores() { public String toString() { return ""; } + + public void setMemory(int memory) { + this.memory = memory; + } + + public void setvCores(int vCores) { + this.vCores = vCores; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index 71a79d4..91251b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -24,10 +24,15 @@ import java.io.IOException; import java.io.StringReader; +import java.io.StringWriter; +import java.net.URI; import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; import javax.ws.rs.core.MediaType; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.Marshaller; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -36,6 +41,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; @@ -50,7 +57,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppSubmissionInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebServicesTestUtils; import org.codehaus.jettison.json.JSONArray; @@ -1763,6 +1773,254 @@ public void testSingleAppKillInvalidId() throws Exception { return; } + @Test + public void testGetNewApplicationIdAndSubmit() throws Exception { + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + amNodeManager.nodeHeartbeat(true); + String mediaTypes[] = { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }; + String id = ""; + for(int i = 0; i < 2; ++i) { + for(String acceptMedia: mediaTypes) { + for(String contentMedia: mediaTypes) { + ClientResponse response = this.constructWebResource("app", "id") + .accept(acceptMedia) + .post(ClientResponse.class); + if(rm.getConfig().getBoolean(YarnConfiguration.YARN_ACL_ENABLE, false) == false) { + assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); + } + else { + if(acceptMedia.equals(MediaType.APPLICATION_XML)) { + assertEquals(Status.OK, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("newApplicationInfo"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + Element element = (Element) nodes.item(0); + id = WebServicesTestUtils.getXmlString(element, "id"); + assertTrue(ConverterUtils.toApplicationId(id) != null); + } + else { + assertEquals(Status.OK, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + JSONObject json = response.getEntity(JSONObject.class); + assertEquals("incorrect number of elements", 2, json.length()); + id = json.getString("id"); + assertTrue(ConverterUtils.toApplicationId(id) != null); + } + switch(i) { + case 0: + testAppSubmit(id, acceptMedia, contentMedia); + break; + case 1: + testAppSubmitErrors(id, acceptMedia, contentMedia); + break; + } + Thread.sleep(500); + } + } + } + } + rm.stop(); + return; + } + + public void testAppSubmit(String appId, String acceptMedia, String contentMedia) throws Exception { + + AppSubmissionInfo appInfo = new AppSubmissionInfo(); + appInfo.setApplicationId(appId); + appInfo.setApplicationName("test"); + appInfo.setPriority(3); + appInfo.setMaxAppAttempts(2); + appInfo.setQueue("testqueue"); + appInfo.setApplicationType("test-type"); + HashMap lr = new HashMap(); + LocalResourceInfo y = new LocalResourceInfo(); + y.setUrl(new URI("http://www.test.com/file.txt")); + y.setSize(100); + y.setTimestamp(System.currentTimeMillis()); + y.setType(LocalResourceType.FILE); + y.setVisibility(LocalResourceVisibility.APPLICATION); + lr.put("example", y); + appInfo.getContainerLaunchContext().setLocalResources(lr); + appInfo.getResource().setMemory(1024); + appInfo.getResource().setvCores(1); + + ClientResponse response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + assertEquals(Status.ACCEPTED, response.getClientResponseStatus()); + assertTrue(response.getHeaders().getFirst("Location").endsWith("/app/" + appId)); + RMApp app = rm.getRMContext().getRMApps().get(ConverterUtils.toApplicationId(appId)); + assertEquals("test", app.getName()); + assertEquals(webserviceUserName, app.getUser()); + assertEquals(2, app.getMaxAppAttempts()); + assertEquals("testqueue", app.getQueue()); + assertEquals("test-type", app.getApplicationType()); + + response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + return; + } + + public void testAppSubmitErrors(String appId, String acceptMedia, String contentMedia) throws Exception { + + AppSubmissionInfo appInfo = new AppSubmissionInfo(); + ClientResponse response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.setApplicationId(appId); + response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.setApplicationName("test"); + response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.setPriority(3); + appInfo.setMaxAppAttempts(2); + appInfo.setQueue("testqueue"); + appInfo.setApplicationType("test-type"); + + response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + // missing local resource info + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + HashMap lr = new HashMap(); + LocalResourceInfo y = new LocalResourceInfo(); + y.setUrl(new URI("http://www.test.com/file.txt")); + y.setSize(100); + y.setTimestamp(System.currentTimeMillis()); + y.setType(LocalResourceType.FILE); + y.setVisibility(LocalResourceVisibility.APPLICATION); + lr.put("example", y); + appInfo.getContainerLaunchContext().setLocalResources(lr); + response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + // missing resource info + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.getResource().setMemory( + rm.getConfig().getInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB) + 1); + appInfo.getResource().setvCores(1); + response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + // bad resource info + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.getResource().setvCores( + rm.getConfig().getInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES) + 1); + appInfo.getResource().setMemory(CONTAINER_MB); + response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + // bad resource info + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + appInfo.getResource().setvCores(0); + appInfo.getResource().setMemory(0); + response = this.constructWebResource("app", appId) + .accept(acceptMedia).entity(appInfo, contentMedia) + .post(ClientResponse.class); + // bad resource info + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + + return; + } + + @Test + public void testAppSubmitBadJsonXML() throws Exception { + + rm.start(); + MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); + amNodeManager.nodeHeartbeat(true); + String id = ""; + + AppSubmissionInfo appInfo = new AppSubmissionInfo(); + appInfo.setApplicationName("test"); + appInfo.setPriority(3); + appInfo.setMaxAppAttempts(2); + appInfo.setQueue("testqueue"); + appInfo.setApplicationType("test-type"); + HashMap lr = new HashMap(); + LocalResourceInfo y = new LocalResourceInfo(); + y.setUrl(new URI("http://www.test.com/file.txt")); + y.setSize(100); + y.setTimestamp(System.currentTimeMillis()); + y.setType(LocalResourceType.FILE); + y.setVisibility(LocalResourceVisibility.APPLICATION); + lr.put("example", y); + appInfo.getContainerLaunchContext().setLocalResources(lr); + appInfo.getResource().setMemory(1024); + appInfo.getResource().setvCores(1); + + JAXBContext context = JAXBContext.newInstance(AppSubmissionInfo.class); + Marshaller m = context.createMarshaller(); + + ClientResponse response; + + for(int i = 0; i < 2; ++i) { + response = this.constructWebResource("app", "id") + .accept(MediaType.APPLICATION_XML) + .post(ClientResponse.class); + if(rm.getConfig().getBoolean(YarnConfiguration.YARN_ACL_ENABLE, false) == false) { + assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus()); + } + else { + assertEquals(Status.OK, response.getClientResponseStatus()); + assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); + String xml = response.getEntity(String.class); + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(xml)); + Document dom = db.parse(is); + NodeList nodes = dom.getElementsByTagName("newApplicationInfo"); + assertEquals("incorrect number of elements", 1, nodes.getLength()); + Element element = (Element) nodes.item(0); + id = WebServicesTestUtils.getXmlString(element, "id"); + assertTrue(ConverterUtils.toApplicationId(id) != null); + appInfo.setApplicationId(id); + StringWriter sw = new StringWriter(); + m.marshal(appInfo, sw); + String body = sw.toString(); + switch(i) { + case 0: + body = body.replace("FILE", "FILES12"); + break; + case 1: + body = body.replace("APPLICATION", "APPLICATION@#"); + break; + } + response = this.constructWebResource("app", id) + .accept(MediaType.APPLICATION_XML).entity(body, MediaType.APPLICATION_XML) + .post(ClientResponse.class); + assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + } + } + rm.stop(); + } + @After @Override public void tearDown() throws Exception {