From 147fb2069d30c1f26689c69704847fc2bd794038 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Fri, 25 Jan 2019 19:33:18 +0530 Subject: [PATCH] YARN-9227 --- .../distributedshell/ApplicationMaster.java | 19 ++++++++++++- .../yarn/applications/distributedshell/Client.java | 9 ++++++- .../distributedshell/TestDistributedShell.java | 31 ++++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 3b58961..ce1160a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -387,8 +387,9 @@ */ public static void main(String[] args) { boolean result = false; + ApplicationMaster appMaster = null; try { - ApplicationMaster appMaster = new ApplicationMaster(); + appMaster = new ApplicationMaster(); LOG.info("Initializing ApplicationMaster"); boolean doRun = appMaster.init(args); if (!doRun) { @@ -400,6 +401,10 @@ public static void main(String[] args) { LOG.error("Error running ApplicationMaster", t); LogManager.shutdown(); ExitUtil.terminate(1, t); + } finally { + if (appMaster != null) { + appMaster.cleanup(); + } } if (result) { LOG.info("Application Master completed successfully. exiting"); @@ -761,6 +766,18 @@ private void printUsage(Options opts) { new HelpFormatter().printHelp("ApplicationMaster", opts); } + private void cleanup() { + Path dst = null; + try { + FileSystem fs = FileSystem.get(conf); + dst = new Path(fs.getHomeDirectory(), getRelativePath(appName, + appId.toString(), "")); + fs.delete(dst, true); + } catch(IOException e) { + LOG.warn("Failed to remove application staging directory {}", dst); + } + } + /** * Main run function for the application master * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 369d94b..6836e38 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -96,6 +96,7 @@ import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.util.timeline.TimelineUtils; +import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -147,6 +148,7 @@ private YarnClient yarnClient; // Application master specific info to register a new Application with RM/ASM private String appName = ""; + private ApplicationId appId; // App master priority private int amPriority = 0; // Queue for App master @@ -746,7 +748,7 @@ public boolean run() throws IOException, YarnException { // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); - ApplicationId appId = appContext.getApplicationId(); + appId = appContext.getApplicationId(); // Set up resource type requirements // For now, both memory and vcores are supported, so we set memory and @@ -1181,6 +1183,11 @@ private void uploadFile(FileSystem fs, String fileSrcPath, fs.copyFromLocalFile(new Path(fileSrcPath), dst); } + @VisibleForTesting + ApplicationId getAppId() { + return appId; + } + private void prepareTimelineDomain() { TimelineClient timelineClient = null; if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 80c1e20..0fd7339 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -1694,4 +1694,35 @@ public void testDistributedShellWithNonExistentFileLocalization() client.init(args); client.run(); } + + + @Test + public void testDistributedShellCleanup() + throws Exception { + String appName = "DistributedShellCleanup"; + String[] args = { + "--jar", + APPMASTER_JAR, + "--num_containers", + "1", + "--shell_command", + Shell.WINDOWS ? "dir" : "ls", + "--appname", + appName + }; + Configuration config = new Configuration(yarnCluster.getConfig()); + Client client = new Client(config); + client.init(args); + boolean result = client.run(); + ApplicationId appId = client.getAppId(); + String relativePath = + ApplicationMaster.getRelativePath(appName, appId.toString(), ""); + FileSystem fs1 = FileSystem.get(config); + Path path = new Path(fs1.getHomeDirectory(), relativePath); + while(fs1.exists(path)) { + Thread.sleep(10); + continue; + } + assertTrue("Distributed Shell Cleanup Fails", !fs1.exists(path)); + } } -- 2.7.4 (Apple Git-66)