diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 1fd1bd2..2005f04 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -318,10 +318,7 @@ public boolean init(String[] args) throws ParseException, IOException { Map envs = System.getenv(); - if (envs.containsKey(ApplicationConstants.AM_APP_ATTEMPT_ID_ENV)) { - appAttemptID = ConverterUtils.toApplicationAttemptId(envs - .get(ApplicationConstants.AM_APP_ATTEMPT_ID_ENV)); - } else if (!envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) { + if (!envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) { if (cliParser.hasOption("app_attempt_id")) { String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); @@ -335,6 +332,23 @@ public boolean init(String[] args) throws ParseException, IOException { appAttemptID = containerId.getApplicationAttemptId(); } + if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) { + throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + + " not set in the environment"); + } + if (!envs.containsKey(ApplicationConstants.NM_HOST_ENV)) { + throw new RuntimeException(ApplicationConstants.NM_HOST_ENV + + " not set in the environment"); + } + if (!envs.containsKey(ApplicationConstants.NM_HTTP_PORT_ENV)) { + throw new RuntimeException(ApplicationConstants.NM_HTTP_PORT_ENV + + " not set in the environment"); + } + if (!envs.containsKey(ApplicationConstants.NM_PORT_ENV)) { + throw new RuntimeException(ApplicationConstants.NM_PORT_ENV + + " not set in the environment"); + } + LOG.info("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java index b9e0625..60e93a9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStreamReader; +import java.net.InetAddress; import java.util.ArrayList; import java.util.EnumSet; import java.util.Map; @@ -41,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; @@ -81,6 +83,8 @@ // set the classpath explicitly private String classpath = null; + private volatile boolean amCompleted = false; + /** * @param args * Command line arguments @@ -179,8 +183,18 @@ public void launchAM(ApplicationAttemptId attemptId) throws IOException { if(!setClasspath && classpath!=null) { envAMList.add("CLASSPATH="+classpath); } - - envAMList.add(ApplicationConstants.AM_APP_ATTEMPT_ID_ENV + "=" + attemptId); + + ContainerId containerId = Records.newRecord(ContainerId.class); + containerId.setApplicationAttemptId(attemptId); + containerId.setId(0); + + String hostname = InetAddress.getLocalHost().getHostName(); + envAMList.add(ApplicationConstants.AM_CONTAINER_ID_ENV + "=" + containerId); + envAMList.add(ApplicationConstants.NM_HOST_ENV + "=" + hostname); + envAMList.add(ApplicationConstants.NM_HTTP_PORT_ENV + "=0"); + envAMList.add(ApplicationConstants.NM_PORT_ENV + "=0"); + envAMList.add(ApplicationConstants.APP_SUBMIT_TIME_ENV + "=" + + System.currentTimeMillis()); String[] envAM = new String[envAMList.size()]; Process amProc = Runtime.getRuntime().exec(amCmd, envAMList.toArray(envAM)); @@ -234,6 +248,7 @@ public void run() { } catch (InterruptedException e) { e.printStackTrace(); } + amCompleted = true; try { // make sure that the error thread exits @@ -306,6 +321,7 @@ public boolean run() throws IOException { appReport = monitorApplication(appId, EnumSet.of( YarnApplicationState.KILLED, YarnApplicationState.FAILED, YarnApplicationState.FINISHED)); + YarnApplicationState appState = appReport.getYarnApplicationState(); FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus(); @@ -341,6 +357,7 @@ public boolean run() throws IOException { private ApplicationReport monitorApplication(ApplicationId appId, Set finalState) throws YarnRemoteException { + int foundAMCompletedCounter = 0; while (true) { // Check app status every 1 second. @@ -370,8 +387,16 @@ private ApplicationReport monitorApplication(ApplicationId appId, return report; } + // wait for 5 seconds after process has completed for app report to + // come back + if (amCompleted) { + ++foundAMCompletedCounter; + if (foundAMCompletedCounter > 5) { + LOG.warn("Waited 5 seconds after process completed for AppReport" + + " to reach desired final state. Not waiting anymore"); + throw new RuntimeException("Failed to receive final AppReport"); + } + } } - } - } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java index 6bb863e..5a91988 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.applications.unmanagedamlauncher; +import static org.junit.Assert.fail; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -91,7 +93,7 @@ private static String getTestRuntimeClasspath() { return envClassPath; } - @Test + @Test(timeout=30000) public void testDSShell() throws Exception { String classpath = getTestRuntimeClasspath(); String javaHome = System.getenv("JAVA_HOME"); @@ -99,7 +101,7 @@ public void testDSShell() throws Exception { LOG.fatal("JAVA_HOME not defined. Test not running."); return; } - // start dist-shell with 0 containers because container launch will fail if + // start dist-shell with 0 containers because container launch will fail if // there are no dist cache resources. String[] args = { "--classpath", @@ -125,4 +127,42 @@ public void testDSShell() throws Exception { } + @Test(timeout=30000) + public void testDSShellError() throws Exception { + String classpath = getTestRuntimeClasspath(); + String javaHome = System.getenv("JAVA_HOME"); + if (javaHome == null) { + LOG.fatal("JAVA_HOME not defined. Test not running."); + return; + } + // start dist-shell with 0 containers because container launch will fail if + // there are no dist cache resources. + // remove shell command to make dist-shell fail + String[] args = { + "--classpath", + classpath, + "--queue", + "default", + "--cmd", + javaHome + + "/bin/java -Xmx512m " + + "org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster " + + "--container_memory 128 --num_containers 0 --priority 0" }; + + LOG.info("Initializing Launcher"); + UnmanagedAMLauncher launcher = new UnmanagedAMLauncher(new Configuration( + yarnCluster.getConfig())); + boolean initSuccess = launcher.init(args); + Assert.assertTrue(initSuccess); + LOG.info("Running Launcher"); + + try { + launcher.run(); + fail("Expected an exception to occur as launch should have failed"); + } catch (RuntimeException e) { + // Expected + } + } + + }