From a8b81d6336e3b317d175765c9b5d047a2f505193 Mon Sep 17 00:00:00 2001 From: Adam Antal Date: Thu, 23 May 2019 18:44:20 +0200 Subject: [PATCH] YARN-9573. DistributedShell can't specify LogAggregationContext --- .../applications/distributedshell/Client.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 08c6b83797f0ddbdfcea4947c14283fd3339e78b..fdbb279c6830fbc34fc0cc9b261f490189cfc797 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -68,6 +68,7 @@ import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; @@ -203,7 +204,9 @@ private String nodeAttributeSpec = ""; // log4j.properties file // if available, add to local resources and set into classpath - private String log4jPropFile = ""; + private String log4jPropFile = ""; + // rolling + private String rollingFilesPattern = ""; // Start time for client private final long clientStartTime = System.currentTimeMillis(); @@ -344,6 +347,8 @@ public Client(Configuration conf) throws Exception { opts.addOption("enforce_execution_type", false, "Flag to indicate whether to enforce execution type of containers"); opts.addOption("log_properties", true, "log4j.properties file"); + opts.addOption("rolling_log_pattern", true, + "pattern for files that should be aggregated in a rolling fashion"); opts.addOption("keep_containers_across_application_attempts", false, "Flag to indicate whether to keep containers across application " + "attempts." @@ -443,6 +448,10 @@ public boolean init(String[] args) throws ParseException { } } + if (cliParser.hasOption("rolling_log_pattern")) { + rollingFilesPattern = cliParser.getOptionValue("rolling_log_pattern"); + } + if (cliParser.hasOption("help")) { printUsage(); return false; @@ -1060,6 +1069,12 @@ public boolean run() throws IOException, YarnException { // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); + if (!rollingFilesPattern.isEmpty()) { + LogAggregationContext logAggregationContext = LogAggregationContext + .newInstance(null, null, rollingFilesPattern, ""); + appContext.setLogAggregationContext(logAggregationContext); + } + // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success -- 2.21.0