diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 3644761..e66a07a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -317,6 +317,7 @@ public boolean init(String[] args) throws ParseException, IOException { "No. of containers on which the shell command needs to be executed"); opts.addOption("priority", true, "Application Priority. Default 0"); opts.addOption("debug", false, "Dump out debug information"); + opts.addOption("log_properties", true, "log4j.properties file"); opts.addOption("help", false, "Print usage"); CommandLine cliParser = new GnuParser().parse(opts, args); @@ -327,6 +328,15 @@ public boolean init(String[] args) throws ParseException, IOException { "No args specified for application master to initialize"); } + if (cliParser.hasOption("log_properties")) { + String log4jPath = cliParser.getOptionValue("log_properties"); + try { + Log4jPropertyHelper.updateLog4jConfiguration(ApplicationMaster.class, log4jPath); + } catch (Exception e) { + LOG.warn("Can not set up custom log4j properties. " + e); + } + } + if (cliParser.hasOption("help")) { printUsage(opts); return false; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 199a16d..9b8356a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -162,6 +162,9 @@ // Command line options private Options opts; + // Hardcoded path to custom log_properties + private final String log4jPath = "log4j.properties"; + /** * @param args Command line arguments */ @@ -253,7 +256,16 @@ public boolean init(String[] args) throws ParseException { if (args.length == 0) { throw new IllegalArgumentException("No args specified for client to initialize"); - } + } + + if (cliParser.hasOption("log_properties")) { + String log4jPath = cliParser.getOptionValue("log_properties"); + try { + Log4jPropertyHelper.updateLog4jConfiguration(Client.class, log4jPath); + } catch (Exception e) { + LOG.warn("Can not set up custom log4j properties. " + e); + } + } if (cliParser.hasOption("help")) { printUsage(); @@ -451,7 +463,8 @@ public boolean run() throws IOException, YarnException { // Set the log4j properties if needed if (!log4jPropFile.isEmpty()) { Path log4jSrc = new Path(log4jPropFile); - Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props"); + String log4jPathSuffix = appName + "/" + appId.getId() + "/" + log4jPath; + Path log4jDst = new Path(fs.getHomeDirectory(), log4jPathSuffix); fs.copyFromLocalFile(false, true, log4jSrc, log4jDst); FileStatus log4jFileStatus = fs.getFileStatus(log4jDst); LocalResource log4jRsrc = Records.newRecord(LocalResource.class); @@ -553,7 +566,9 @@ public boolean run() throws IOException, YarnException { if (debugFlag) { vargs.add("--debug"); } - + if (!log4jPropFile.isEmpty()) { + vargs.add("--log_properties " + log4jPath); + } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Log4jPropertyHelper.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Log4jPropertyHelper.java new file mode 100644 index 0000000..cbfe16c --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Log4jPropertyHelper.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.applications.distributedshell; + +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Map.Entry; +import java.util.Properties; + +import org.apache.commons.io.IOUtils; +import org.apache.log4j.LogManager; +import org.apache.log4j.PropertyConfigurator; + + +public class Log4jPropertyHelper { + + public static void updateLog4jConfiguration(Class targetClass, + String log4jPath) throws Exception { + Properties customProperties = new Properties(); + FileInputStream fs = null; + InputStream is = null; + try { + fs = new FileInputStream(log4jPath); + is = targetClass.getResourceAsStream("/log4j.properties"); + customProperties.load(fs); + Properties originalProperties = new Properties(); + originalProperties.load(is); + for (Entry entry : customProperties.entrySet()) { + originalProperties.setProperty(entry.getKey().toString(), entry + .getValue().toString()); + } + LogManager.resetConfiguration(); + PropertyConfigurator.configure(originalProperties); + }finally { + IOUtils.closeQuietly(is); + IOUtils.closeQuietly(fs); + } + } +} diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 2f311b5..3cbe767 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -18,11 +18,14 @@ package org.apache.hadoop.yarn.applications.distributedshell; +import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; +import java.io.FileReader; import java.io.IOException; import java.io.OutputStream; +import java.io.PrintWriter; import java.net.URL; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; @@ -306,5 +309,111 @@ public void testContainerLaunchFailureHandling() throws Exception { } + @Test(timeout=90000) + public void testDSShellWithCustomLogPropertyFile() throws Exception { + final File basedir = + new File("target", TestDistributedShell.class.getName()); + final File tmpDir = new File(basedir, "tmpDir"); + tmpDir.mkdirs(); + final File customLogProperty = new File(tmpDir, "custom_log4j.properties"); + if (customLogProperty.exists()) { + customLogProperty.delete(); + } + if(!customLogProperty.createNewFile()) { + Assert.fail("Can not create custom log4j property file."); + } + PrintWriter fileWriter = new PrintWriter(customLogProperty); + // set the output to DEBUG level + fileWriter.write("log4j.rootLogger=debug,stdout"); + fileWriter.close(); + String[] args = { + "--jar", + APPMASTER_JAR, + "--num_containers", + "3", + "--shell_command", + "echo", + "--shell_args", + "HADOOP", + "--log_properties", + customLogProperty.getAbsolutePath(), + "--master_memory", + "512", + "--master_vcores", + "2", + "--container_memory", + "128", + "--container_vcores", + "1" + }; + + //Before run the DS, the default the log level is INFO + final Log LOG_Client = + LogFactory.getLog(Client.class); + Assert.assertTrue(LOG_Client.isInfoEnabled()); + Assert.assertFalse(LOG_Client.isDebugEnabled()); + final Log LOG_AM = LogFactory.getLog(ApplicationMaster.class); + Assert.assertTrue(LOG_AM.isInfoEnabled()); + Assert.assertFalse(LOG_AM.isDebugEnabled()); + + LOG.info("Initializing DS Client"); + final Client client = + new Client(new Configuration(yarnCluster.getConfig())); + boolean initSuccess = client.init(args); + Assert.assertTrue(initSuccess); + LOG.info("Running DS Client"); + boolean result = client.run(); + LOG.info("Client run completed. Result=" + result); + Assert.assertTrue(countExpectedWordFromAMLog(3, "DEBUG") > 10); + //After DS is finished, the log level should be DEBUG + Assert.assertTrue(LOG_Client.isInfoEnabled()); + Assert.assertTrue(LOG_Client.isDebugEnabled()); + Assert.assertTrue(LOG_AM.isInfoEnabled()); + Assert.assertTrue(LOG_AM.isDebugEnabled()); + } + + private int countExpectedWordFromAMLog(int containerNum, String expectedWord) { + File logFolder = + new File(yarnCluster.getNodeManager(0).getConfig() + .get(YarnConfiguration.NM_LOG_DIRS, + YarnConfiguration.DEFAULT_NM_LOG_DIRS)); + + File[] listOfFiles = logFolder.listFiles(); + int currentContainerLogFileIndex = -1; + for (int i = 0; i < listOfFiles.length; i++) { + if (listOfFiles[i].listFiles().length == containerNum + 1) { + currentContainerLogFileIndex = i; + break; + } + } + Assert.assertTrue(currentContainerLogFileIndex != -1); + File[] containerLogs = + listOfFiles[currentContainerLogFileIndex].listFiles(); + int numOfWords = 0; + for (File containerLog : containerLogs) { + for (File output : containerLog.listFiles()) { + BufferedReader br = null; + try { + String sCurrentLine; + br = new BufferedReader(new FileReader(output)); + while ((sCurrentLine = br.readLine()) != null) { + if (sCurrentLine.contains(expectedWord)) { + numOfWords++; + } + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + if (br != null) + br.close(); + } catch (IOException ex) { + ex.printStackTrace(); + } + } + } + } + return numOfWords; + } }