diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 88ae207..99da9be 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -27,6 +27,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.ByteBuffer; +import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -236,7 +237,7 @@ // Location of shell script ( obtained from info set in env ) // Shell script path in fs - private String shellScriptPath = ""; + private String scriptPath = ""; // Timestamp needed for creating a local resource private long shellScriptPathTimestamp = 0; // File length needed for local resource @@ -451,7 +452,7 @@ public boolean init(String[] args) throws ParseException, IOException { } if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION)) { - shellScriptPath = envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION); + scriptPath = envs.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION); if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP)) { shellScriptPathTimestamp = Long.valueOf(envs @@ -462,10 +463,10 @@ public boolean init(String[] args) throws ParseException, IOException { .get(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN)); } - if (!shellScriptPath.isEmpty() + if (!scriptPath.isEmpty() && (shellScriptPathTimestamp <= 0 || shellScriptPathLen <= 0)) { LOG.error("Illegal values in env for shell script path" + ", path=" - + shellScriptPath + ", len=" + shellScriptPathLen + ", timestamp=" + + scriptPath + ", len=" + shellScriptPathLen + ", timestamp=" + shellScriptPathTimestamp); throw new IllegalArgumentException( "Illegal values in env for shell script path"); @@ -901,30 +902,25 @@ public void run() { // resources too. // In this scenario, if a shell script is specified, we need to have it // copied and made available to the container. - if (!shellScriptPath.isEmpty()) { - Path renamedSchellScriptPath = null; + if (!scriptPath.isEmpty()) { + Path renamedScriptPath = null; if (Shell.WINDOWS) { - renamedSchellScriptPath = new Path(shellScriptPath + ".bat"); + renamedScriptPath = new Path(scriptPath + ".bat"); } else { - renamedSchellScriptPath = new Path(shellScriptPath + ".sh"); - } - try { - FileSystem fs = renamedSchellScriptPath.getFileSystem(conf); - fs.rename(new Path(shellScriptPath), renamedSchellScriptPath); - } catch (IOException e) { - LOG.warn("Not able to add suffix (.bat/.sh) to the shell script filename"); - throw new YarnRuntimeException(e); + renamedScriptPath = new Path(scriptPath + ".sh"); } + // rename the script file based on the underlying OS syntax. + renameScriptFile(renamedScriptPath); LocalResource shellRsrc = Records.newRecord(LocalResource.class); shellRsrc.setType(LocalResourceType.FILE); shellRsrc.setVisibility(LocalResourceVisibility.APPLICATION); try { shellRsrc.setResource(ConverterUtils.getYarnUrlFromURI(new URI( - renamedSchellScriptPath.toString()))); + renamedScriptPath.toString()))); } catch (URISyntaxException e) { LOG.error("Error when trying to use shell script path specified" - + " in env, path=" + renamedSchellScriptPath); + + " in env, path=" + renamedScriptPath); e.printStackTrace(); // A failure scenario on bad input such as invalid shell script path @@ -949,7 +945,7 @@ public void run() { // Set executable command vargs.add(shellCommand); // Set shell script path - if (!shellScriptPath.isEmpty()) { + if (!scriptPath.isEmpty()) { vargs.add(Shell.WINDOWS ? ExecBatScripStringtPath : ExecShellStringPath); } @@ -983,6 +979,30 @@ public void run() { } } + private void renameScriptFile(final Path renamedScriptPath) { + String jobUserName = + System.getenv(ApplicationConstants.Environment.USER.name()); + UserGroupInformation jobUserUgi = + UserGroupInformation.createRemoteUser(jobUserName); + + jobUserUgi.doAs(new PrivilegedAction() { + @Override + public Void run() { + try { + FileSystem fs = renamedScriptPath.getFileSystem(conf); + fs.rename(new Path(scriptPath), renamedScriptPath); + } catch (IOException e) { + LOG.warn("Not able to add suffix (.bat/.sh) to the shell script filename"); + throw new YarnRuntimeException(e); + } + return null; + } + }); + LOG.info("User " + jobUserName + + " added suffix(.sh/.bat) to script file as " + renamedScriptPath); + } + + /** * Setup the request that will be sent to the RM for the container ask. *