diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 3644761..c380832 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -82,6 +82,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Log4jPropertyHelper; import org.apache.hadoop.yarn.util.Records; /** @@ -317,6 +318,7 @@ public boolean init(String[] args) throws ParseException, IOException { "No. of containers on which the shell command needs to be executed"); opts.addOption("priority", true, "Application Priority. Default 0"); opts.addOption("debug", false, "Dump out debug information"); + opts.addOption("log_properties", true, "log4j.properties file"); opts.addOption("help", false, "Print usage"); CommandLine cliParser = new GnuParser().parse(opts, args); @@ -327,6 +329,15 @@ public boolean init(String[] args) throws ParseException, IOException { "No args specified for application master to initialize"); } + if (cliParser.hasOption("log_properties")) { + String log4jPath = cliParser.getOptionValue("log_properties"); + try { + Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, log4jPath); + } catch (Exception e) { + LOG.warn("Can not set up custom log4j properties. " + e); + } + } + if (cliParser.hasOption("help")) { printUsage(opts); return false; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 199a16d..298f24a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -71,6 +71,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Log4jPropertyHelper; import org.apache.hadoop.yarn.util.Records; /** @@ -162,6 +163,9 @@ // Command line options private Options opts; + // Hardcoded path to custom log_properties + private final String log4jPath = "log4j.properties"; + /** * @param args Command line arguments */ @@ -182,7 +186,7 @@ public static void main(String[] args) { } result = client.run(); } catch (Throwable t) { - LOG.fatal("Error running CLient", t); + LOG.info("Error running CLient", t); System.exit(1); } if (result) { @@ -253,7 +257,16 @@ public boolean init(String[] args) throws ParseException { if (args.length == 0) { throw new IllegalArgumentException("No args specified for client to initialize"); - } + } + + if (cliParser.hasOption("log_properties")) { + String log4jPath = cliParser.getOptionValue("log_properties"); + try { + Log4jPropertyHelper.updateLog4jConfiguration(Client.class, log4jPath); + } catch (Exception e) { + LOG.warn("Can not set up custom log4j properties. " + e); + } + } if (cliParser.hasOption("help")) { printUsage(); @@ -451,7 +464,7 @@ public boolean run() throws IOException, YarnException { // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { Path log4jSrc = new Path(log4jPropFile); - Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props"); + Path log4jDst = new Path(fs.getHomeDirectory(), log4jPath); fs.copyFromLocalFile(false, true, log4jSrc, log4jDst); FileStatus log4jFileStatus = fs.getFileStatus(log4jDst); LocalResource log4jRsrc = Records.newRecord(LocalResource.class); @@ -553,7 +566,9 @@ public boolean run() throws IOException, YarnException { if (debugFlag) { vargs.add("--debug"); } - + if (!log4jPropFile.isEmpty()) { + vargs.add("--log_properties " + log4jPath); + } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Log4jPropertyHelper.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Log4jPropertyHelper.java new file mode 100644 index 0000000..acacec9 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Log4jPropertyHelper.java @@ -0,0 +1,28 @@ +package org.apache.hadoop.yarn.util; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Properties; + +import org.apache.log4j.LogManager; +import org.apache.log4j.PropertyConfigurator; + + +public class Log4jPropertyHelper { + + public static void updateLog4jConfiguration(Class targetClass, + String log4jPath) throws Exception { + Properties customProperties = new Properties(); + customProperties.load(new FileInputStream(log4jPath)); + Properties originalProperties = new Properties(); + originalProperties.load(targetClass + .getResourceAsStream("/log4j.properties")); + for (Object key : customProperties.keySet()) { + originalProperties.setProperty(key.toString(), customProperties.get(key) + .toString()); + } + LogManager.resetConfiguration(); + PropertyConfigurator.configure(originalProperties); + } +}