diff --git hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 24f5ff1..f4c89f6 100644 --- hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -41,7 +41,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -49,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -59,7 +59,7 @@ private static final Log LOG = LogFactory.getLog(ResourceMgrDelegate.class); private YarnConfiguration conf; - private GetNewApplicationResponse application; + private ApplicationSubmissionContext application; private ApplicationId applicationId; @Private @VisibleForTesting @@ -178,7 +178,7 @@ public String getFilesystemName() throws IOException, InterruptedException { public JobID getNewJobID() throws IOException, InterruptedException { try { - this.application = client.getNewApplication(); + this.application = client.createApplication().getApplicationSubmissionContext(); this.applicationId = this.application.getApplicationId(); return TypeConverter.fromYarn(applicationId); } catch (YarnException e) { @@ -272,9 +272,9 @@ public ApplicationId getApplicationId() { } @Override - public GetNewApplicationResponse getNewApplication() throws YarnException, - IOException { - return client.getNewApplication(); + public YarnClientApplication createApplication() throws + YarnException, IOException { + return client.createApplication(); } @Override diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 9e14ca4..4350725 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -61,6 +61,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; @@ -217,7 +218,6 @@ public Client() throws Exception { /** * Helper function to print out usage - * @param opts Parsed command line options */ private void printUsage() { new HelpFormatter().printHelp("Client", opts); @@ -351,16 +351,15 @@ public boolean run() throws IOException, YarnException { } } - // Get a new application id - GetNewApplicationResponse newApp = yarnClient.getNewApplication(); - ApplicationId appId = newApp.getApplicationId(); - + // Get a new application id + YarnClientApplication app = yarnClient.createApplication(); + GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); // TODO get min/max resource capabilities from RM and change memory ask if needed // If we do not have min/max, we may not be able to correctly request // the required resources from the RM for the app master // Memory ask has to be a multiple of min and less than max. // Dump out information about cluster capability as seen by the resource manager - int maxMem = newApp.getMaximumResourceCapability().getMemory(); + int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. @@ -371,13 +370,9 @@ public boolean run() throws IOException, YarnException { amMemory = maxMem; } - // Create launch context for app master - LOG.info("Setting up application submission context for ASM"); - ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); - - // set the application id - appContext.setApplicationId(appId); // set the application name + ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); + ApplicationId appId = appContext.getApplicationId(); appContext.setApplicationName(appName); // Set up the container launch context for the application master diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java index b209d95..fbfe506 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java @@ -274,17 +274,12 @@ public boolean run() throws IOException, YarnException { // Connect to ResourceManager rmClient.start(); try { - // Get a new application id - GetNewApplicationResponse newApp = rmClient.getNewApplication(); - ApplicationId appId = newApp.getApplicationId(); - // Create launch context for app master LOG.info("Setting up application submission context for ASM"); - ApplicationSubmissionContext appContext = Records - .newRecord(ApplicationSubmissionContext.class); - - // set the application id - appContext.setApplicationId(appId); + ApplicationSubmissionContext appContext = rmClient.createApplication() + .getApplicationSubmissionContext(); + ApplicationId appId = appContext.getApplicationId(); + // set the application name appContext.setApplicationName(appName); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 751c4ba..869a52b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -28,7 +28,6 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.Text; import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -79,27 +78,18 @@ protected YarnClient(String name) { /** *
- * Obtain a new {@link ApplicationId} for submitting new applications. + * Obtain a {@link YarnClientApplication} for a new application, + * which in turn contains the {@link ApplicationSubmissionContext} and + * {@link org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse} + * objects. *
- * - *- * Returns a response which contains {@link ApplicationId} that can be used to - * submit a new application. See - * {@link #submitApplication(ApplicationSubmissionContext)}. - *
- * - *- * See {@link GetNewApplicationResponse} for other information that is - * returned. - *
- * - * @return response containing the newApplicationId to be used
- * to submit an application
+ *
+ * @return {@link YarnClientApplication} built for a new application
* @throws YarnException
* @throws IOException
*/
- public abstract GetNewApplicationResponse getNewApplication() throws YarnException,
- IOException;
+ public abstract YarnClientApplication createApplication()
+ throws YarnException, IOException;
/**
* @@ -114,7 +104,7 @@ public abstract GetNewApplicationResponse getNewApplication() throws YarnExcepti * @return {@link ApplicationId} of the accepted application * @throws YarnException * @throws IOException - * @see #getNewApplication() + * @see #createApplication() */ public abstract ApplicationId submitApplication(ApplicationSubmissionContext appContext) throws YarnException, IOException; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClientApplication.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClientApplication.java new file mode 100644 index 0000000..7f9f473 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClientApplication.java @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; + +@InterfaceAudience.Public +@InterfaceStability.Stable + +/** + * Holder for the {@link GetNewApplicationResponse} and {@link + * ApplicationSubmissionContext} objects created via {@link org.apache.hadoop + * .yarn.client.api.YarnClient#createApplication()} + */ +public class YarnClientApplication { + private final GetNewApplicationResponse newAppResponse; + private final ApplicationSubmissionContext appSubmissionContext; + + public YarnClientApplication(GetNewApplicationResponse newAppResponse, + ApplicationSubmissionContext appContext) { + this.newAppResponse = newAppResponse; + this.appSubmissionContext = appContext; + } + + public GetNewApplicationResponse getNewApplicationResponse() { + return newAppResponse; + } + + public ApplicationSubmissionContext getApplicationSubmissionContext() { + return appSubmissionContext; + } +} + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 3e1d579..cbe8120 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.YarnRPC; @@ -125,8 +126,7 @@ protected void serviceStop() throws Exception { super.serviceStop(); } - @Override - public GetNewApplicationResponse getNewApplication() + private GetNewApplicationResponse getNewApplication() throws YarnException, IOException { GetNewApplicationRequest request = Records.newRecord(GetNewApplicationRequest.class); @@ -134,6 +134,17 @@ public GetNewApplicationResponse getNewApplication() } @Override + public YarnClientApplication createApplication() + throws YarnException, IOException { + ApplicationSubmissionContext context = Records.newRecord + (ApplicationSubmissionContext.class); + GetNewApplicationResponse newApp = getNewApplication(); + ApplicationId appId = newApp.getApplicationId(); + context.setApplicationId(appId); + return new YarnClientApplication(newApp, context); + } + + @Override public ApplicationId submitApplication(ApplicationSubmissionContext appContext) throws YarnException, IOException { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java index 9398f98..5955f26 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java @@ -39,7 +39,6 @@ import org.apache.hadoop.yarn.api.ApplicationMasterProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -61,7 +60,6 @@ import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; import org.apache.hadoop.yarn.client.api.AMRMClient.StoredContainerRequest; -import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.MiniYARNCluster; @@ -117,13 +115,9 @@ public static void setup() throws Exception { @Before public void startApp() throws Exception { // submit new app - GetNewApplicationResponse newApp = yarnClient.getNewApplication(); - ApplicationId appId = newApp.getApplicationId(); - - ApplicationSubmissionContext appContext = Records - .newRecord(ApplicationSubmissionContext.class); - // set the application id - appContext.setApplicationId(appId); + ApplicationSubmissionContext appContext = + yarnClient.createApplication().getApplicationSubmissionContext(); + ApplicationId appId = appContext.getApplicationId(); // set the application name appContext.setApplicationName("Test"); // Set the priority for the application master diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java index 309b5af..5bcb428 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java @@ -36,7 +36,6 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -58,9 +57,6 @@ import org.apache.hadoop.yarn.client.api.NMClient; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest; -import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl; -import org.apache.hadoop.yarn.client.api.impl.NMClientImpl; -import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.MiniYARNCluster; @@ -102,13 +98,9 @@ public void setup() throws YarnException, IOException { nodeReports = yarnClient.getNodeReports(); // submit new app - GetNewApplicationResponse newApp = yarnClient.getNewApplication(); - ApplicationId appId = newApp.getApplicationId(); - - ApplicationSubmissionContext appContext = Records - .newRecord(ApplicationSubmissionContext.class); - // set the application id - appContext.setApplicationId(appId); + ApplicationSubmissionContext appContext = + yarnClient.createApplication().getApplicationSubmissionContext(); + ApplicationId appId = appContext.getApplicationId(); // set the application name appContext.setApplicationName("Test"); // Set the priority for the application master