From d26e61e9f3af5b786e5cdbc17b6aa7808528feeb Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Fri, 2 Aug 2019 17:39:20 +0530 Subject: [PATCH] YARN-8199. Logging fileSize of log files under NM Local Dir. --- .../org/apache/hadoop/yarn/conf/YarnConfiguration.java | 5 +++++ .../src/main/resources/yarn-default.xml | 8 ++++++++ .../logaggregation/AppLogAggregatorImpl.java | 18 +++++++++++++++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d8dc7c0..dca577d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1342,6 +1342,11 @@ public static boolean isAclEnabled(Configuration conf) { public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX + "log-aggregation.retain-seconds"; public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1; + + public static final String LOG_AGGREGATION_DEBUG_FILESIZE = YARN_PREFIX + + "log-aggregation.debug.filesize"; + public static final long DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE + = 100 * 1024 * 1024; /** * How long to wait between aggregated log retention checks. If set to diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 76ae2be..93ba5f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1279,6 +1279,14 @@ + The log files created under NM Local Directories + will be logged if it exceeds the configured bytes. This + only takes effect if log4j level is at least Debug. + yarn.log-aggregation.debug.filesize + 104857600 + + + Specify which log file controllers we will support. The first file controller we add will be used to write the aggregated logs. This comma separated configuration will work with the configuration: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index 04503ef..84eec4f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -109,7 +109,7 @@ private final AtomicBoolean waiting = new AtomicBoolean(false); private int logAggregationTimes = 0; private int cleanupOldLogTimes = 0; - + private long logFileSizeThreshold; private boolean renameTemporaryLogFileFailed = false; private final Map containerLogAggregators = @@ -176,6 +176,9 @@ public AppLogAggregatorImpl(Dispatcher dispatcher, this.nodeId = nodeId; this.logAggPolicy = getLogAggPolicy(conf); this.recoveredLogInitedTime = recoveredLogInitedTime; + this.logFileSizeThreshold = + conf.getLong(YarnConfiguration.LOG_AGGREGATION_DEBUG_FILESIZE, + YarnConfiguration.DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE); if (logAggregationFileController == null) { // by default, use T-File Controller this.logAggregationFileController = new LogAggregationTFileController(); @@ -330,6 +333,19 @@ private void uploadLogsForContainers(boolean appFinished) uploadedLogsInThisCycle = true; List uploadedFilePathsInThisCycleList = new ArrayList<>(); uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle); + if (LOG.isDebugEnabled()) { + for (Path uploadedFilePath : uploadedFilePathsInThisCycleList) { + try { + long fileSize = lfs.getFileStatus(uploadedFilePath).getLen(); + if (fileSize >= logFileSizeThreshold) { + LOG.debug("Log File " + uploadedFilePath + + " size is " + fileSize + " bytes"); + } + } catch (Exception e1) { + LOG.error("Failed to get log file size " + e1); + } + } + } deletionTask = new FileDeletionTask(delService, this.userUgi.getShortUserName(), null, uploadedFilePathsInThisCycleList); -- 2.7.4 (Apple Git-66)