From c772248bfbad420cc6ed3906474d881620c1de06 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Wed, 17 Apr 2019 12:47:06 +0530 Subject: [PATCH] YARN-9482 --- .../distributedshell/ApplicationMaster.java | 14 +++++--------- .../yarn/applications/distributedshell/Client.java | 18 ++++-------------- .../distributedshell/TestDistributedShell.java | 4 +--- 3 files changed, 10 insertions(+), 26 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index f4d061a..3d0b830 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -366,6 +366,7 @@ private int yarnShellIdCounter = 1; private final AtomicLong allocIdCounter = new AtomicLong(1); + private static final String STAGING_AREA = "/tmp/distributedshell"; @VisibleForTesting protected final Set launchedContainers = @@ -777,8 +778,7 @@ private void cleanup() { Path dst = null; try { FileSystem fs = FileSystem.get(conf); - dst = new Path(fs.getHomeDirectory(), getRelativePath(appName, - appId.toString(), "")); + dst = getStagingDir(appId.toString()); fs.delete(dst, true); } catch(IOException e) { LOG.warn("Failed to remove application staging directory {}", dst); @@ -1050,9 +1050,8 @@ protected boolean finish() { return success; } - public static String getRelativePath(String appName, - String appId, String fileDstPath) { - return appName + "/" + appId + "/" + fileDstPath; + public static Path getStagingDir(String appId) { + return new Path(STAGING_AREA + Path.SEPARATOR + appId + Path.SEPARATOR); } @VisibleForTesting @@ -1480,10 +1479,7 @@ public void run() { localizableFiles.stream().forEach(fileName -> { try { - String relativePath = - getRelativePath(appName, appId.toString(), fileName); - Path dst = - new Path(fs.getHomeDirectory(), relativePath); + Path dst = new Path(getStagingDir(appId.toString()), fileName); FileStatus fileStatus = fs.getFileStatus(dst); LocalResource localRes = LocalResource.newInstance( URL.fromURI(dst.toUri()), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 08c6b83..e0c7695 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -857,12 +857,8 @@ public boolean run() throws IOException, YarnException { long hdfsShellScriptTimestamp = 0; if (!shellScriptPath.isEmpty()) { Path shellSrc = new Path(shellScriptPath); - String shellPathSuffix = - ApplicationMaster.getRelativePath(appName, - applicationId.toString(), - SCRIPT_PATH); - Path shellDst = - new Path(fs.getHomeDirectory(), shellPathSuffix); + Path shellDst = new Path(ApplicationMaster.getStagingDir( + applicationId.toString()), SCRIPT_PATH); fs.copyFromLocalFile(false, true, shellSrc, shellDst); hdfsShellScriptLocation = shellDst.toUri().toString(); FileStatus shellFileStatus = fs.getFileStatus(shellDst); @@ -1167,10 +1163,7 @@ private void forceKillApplication(ApplicationId appId) private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId, Map localResources, String resources) throws IOException { - String suffix = - ApplicationMaster.getRelativePath(appName, appId, fileDstPath); - Path dst = - new Path(fs.getHomeDirectory(), suffix); + Path dst = new Path(ApplicationMaster.getStagingDir(appId), fileDstPath); if (fileSrcPath == null) { FSDataOutputStream ostream = null; try { @@ -1194,10 +1187,7 @@ private void addToLocalResources(FileSystem fs, String fileSrcPath, private void uploadFile(FileSystem fs, String fileSrcPath, String fileDstPath, String appId) throws IOException { - String relativePath = - ApplicationMaster.getRelativePath(appName, appId, fileDstPath); - Path dst = - new Path(fs.getHomeDirectory(), relativePath); + Path dst = new Path(ApplicationMaster.getStagingDir(appId), fileDstPath); LOG.info("Uploading file: " + fileSrcPath + " to " + dst); fs.copyFromLocalFile(new Path(fileSrcPath), dst); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 025e543..db1e3f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -1831,10 +1831,8 @@ public void testDistributedShellCleanup() client.init(args); client.run(); ApplicationId appId = client.getAppId(); - String relativePath = - ApplicationMaster.getRelativePath(appName, appId.toString(), ""); FileSystem fs1 = FileSystem.get(config); - Path path = new Path(fs1.getHomeDirectory(), relativePath); + Path path = ApplicationMaster.getStagingDir(appId.toString()); GenericTestUtils.waitFor(() -> { try { -- 2.7.4 (Apple Git-66)