diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 3644761..c380832 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -82,6 +82,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Log4jPropertyHelper; import org.apache.hadoop.yarn.util.Records; /** @@ -317,6 +318,7 @@ public boolean init(String[] args) throws ParseException, IOException { "No. of containers on which the shell command needs to be executed"); opts.addOption("priority", true, "Application Priority. Default 0"); opts.addOption("debug", false, "Dump out debug information"); + opts.addOption("log_properties", true, "log4j.properties file"); opts.addOption("help", false, "Print usage"); CommandLine cliParser = new GnuParser().parse(opts, args); @@ -327,6 +329,15 @@ public boolean init(String[] args) throws ParseException, IOException { "No args specified for application master to initialize"); } + if (cliParser.hasOption("log_properties")) { + String log4jPath = cliParser.getOptionValue("log_properties"); + try { + Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, log4jPath); + } catch (Exception e) { + LOG.warn("Can not set up custom log4j properties. " + e); + } + } + if (cliParser.hasOption("help")) { printUsage(opts); return false; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 199a16d..21f7508 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -71,6 +71,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Log4jPropertyHelper; import org.apache.hadoop.yarn.util.Records; /** @@ -162,6 +163,9 @@ // Command line options private Options opts; + // Hardcoded path to custom log_properties + private final String log4jPath = "log4j.properties"; + /** * @param args Command line arguments */ @@ -253,7 +257,16 @@ public boolean init(String[] args) throws ParseException { if (args.length == 0) { throw new IllegalArgumentException("No args specified for client to initialize"); - } + } + + if (cliParser.hasOption("log_properties")) { + String log4jPath = cliParser.getOptionValue("log_properties"); + try { + Log4jPropertyHelper.updateLog4jConfiguration(Client.class, log4jPath); + } catch (Exception e) { + LOG.warn("Can not set up custom log4j properties. " + e); + } + } if (cliParser.hasOption("help")) { printUsage(); @@ -451,7 +464,7 @@ public boolean run() throws IOException, YarnException { // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { Path log4jSrc = new Path(log4jPropFile); - Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props"); + Path log4jDst = new Path(fs.getHomeDirectory(), log4jPath); fs.copyFromLocalFile(false, true, log4jSrc, log4jDst); FileStatus log4jFileStatus = fs.getFileStatus(log4jDst); LocalResource log4jRsrc = Records.newRecord(LocalResource.class); @@ -553,7 +566,9 @@ public boolean run() throws IOException, YarnException { if (debugFlag) { vargs.add("--debug"); } - + if (!log4jPropFile.isEmpty()) { + vargs.add("--log_properties " + log4jPath); + } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Log4jPropertyHelper.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Log4jPropertyHelper.java new file mode 100644 index 0000000..8383350 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Log4jPropertyHelper.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Map.Entry; +import java.util.Properties; + +import org.apache.commons.io.IOUtils; +import org.apache.log4j.LogManager; +import org.apache.log4j.PropertyConfigurator; + + +public class Log4jPropertyHelper { + + public static void updateLog4jConfiguration(Class targetClass, + String log4jPath) throws Exception { + Properties customProperties = new Properties(); + FileInputStream fs = null; + InputStream is = null; + try { + fs = new FileInputStream(log4jPath); + is = targetClass.getResourceAsStream("/log4j.properties"); + customProperties.load(fs); + Properties originalProperties = new Properties(); + originalProperties.load(is); + for (Entry entry : customProperties.entrySet()) { + originalProperties.setProperty(entry.getKey().toString(), entry + .getValue().toString()); + } + LogManager.resetConfiguration(); + PropertyConfigurator.configure(originalProperties); + }finally { + IOUtils.closeQuietly(is); + IOUtils.closeQuietly(fs); + } + } +}