From 3fc1b1cfbc7ca247b850bfe3e63890c04b578bc8 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Fri, 31 May 2019 16:27:18 +0530 Subject: [PATCH] YARN-8199. Logging fileSize of log files under NM Local Dir. --- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 5 +++++ .../src/main/resources/yarn-default.xml | 7 +++++++ .../logaggregation/AppLogAggregatorImpl.java | 19 ++++++++++++++++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 6bbcdcb..3867439 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1370,6 +1370,11 @@ public static boolean isAclEnabled(Configuration conf) { public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX + "log-aggregation.retain-seconds"; public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1; + + public static final String LOG_AGGREGATION_DEBUG_FILESIZE_BYTES = YARN_PREFIX + + "log-aggregation.debug.filesize.bytes"; + public static final long DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE_BYTES + = 100 * 1024 * 1024; /** * How long to wait between aggregated log retention checks. If set to diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 9741f6c..a40ad1a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1292,6 +1292,13 @@ + The log files created under NM Local Directories + will be logged if it exceeds the configured bytes. + yarn.log-aggregation.debug.filesize.bytes + 104857600 + + + Specify which log file controllers we will support. The first file controller we add will be used to write the aggregated logs. This comma separated configuration will work with the configuration: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index fdac2e4..1af7a18 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -109,7 +109,7 @@ private final AtomicBoolean waiting = new AtomicBoolean(false); private int logAggregationTimes = 0; private int cleanupOldLogTimes = 0; - + private long logFileSizeThreshold; private boolean renameTemporaryLogFileFailed = false; private final Map containerLogAggregators = @@ -176,6 +176,9 @@ public AppLogAggregatorImpl(Dispatcher dispatcher, this.nodeId = nodeId; this.logAggPolicy = getLogAggPolicy(conf); this.recoveredLogInitedTime = recoveredLogInitedTime; + this.logFileSizeThreshold = + conf.getLong(YarnConfiguration.LOG_AGGREGATION_DEBUG_FILESIZE_BYTES, + YarnConfiguration.DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE_BYTES); if (logAggregationFileController == null) { // by default, use T-File Controller this.logAggregationFileController = new LogAggregationTFileController(); @@ -330,6 +333,20 @@ private void uploadLogsForContainers(boolean appFinished) uploadedLogsInThisCycle = true; List uploadedFilePathsInThisCycleList = new ArrayList<>(); uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle); + if (LOG.isDebugEnabled()) { + try { + long fileSize; + for(Path uploadedFilePath : uploadedFilePathsInThisCycleList) { + fileSize = lfs.getFileStatus(uploadedFilePath).getLen(); + if(fileSize >= logFileSizeThreshold) { + LOG.debug("Log File " + uploadedFilePath + + "size is " + fileSize + " bytes"); + } + } + } catch(Exception e1) { + LOG.error("Failed to get log file size " + e1); + } + } deletionTask = new FileDeletionTask(delService, this.userUgi.getShortUserName(), null, uploadedFilePathsInThisCycleList); -- 2.7.4 (Apple Git-66)