From 854f7f9bb6f6fb3c638d40359e74a25465f26513 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Thu, 10 Jan 2019 22:56:51 +0530 Subject: [PATCH] YARN-8199 --- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++++ .../src/main/resources/yarn-default.xml | 7 +++++++ .../logaggregation/AppLogAggregatorImpl.java | 19 ++++++++++++++++++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index c29707c..d125b5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1348,6 +1348,10 @@ public static boolean isAclEnabled(Configuration conf) { public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX + "log-aggregation.retain-seconds"; public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1; + + public static final String LOG_AGGREGATION_DEBUG_FILESIZE = YARN_PREFIX + + "log-aggregation.debug.filesize"; + public static final long DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE = 104857600; /** * How long to wait between aggregated log retention checks. If set to diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index a0e0eda..5dca516 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1284,6 +1284,13 @@ + The log files created under NM Local Directories + will be logged if it exceeds the configured bytes. + yarn.log-aggregation.debug.filesize + 104857600 + + + Specify which log file controllers we will support. The first file controller we add will be used to write the aggregated logs. This comma separated configuration will work with the configuration: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index 04503ef..86491e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -109,7 +109,7 @@ private final AtomicBoolean waiting = new AtomicBoolean(false); private int logAggregationTimes = 0; private int cleanupOldLogTimes = 0; - + private static long logFileSizeThreshold; private boolean renameTemporaryLogFileFailed = false; private final Map containerLogAggregators = @@ -176,6 +176,9 @@ public AppLogAggregatorImpl(Dispatcher dispatcher, this.nodeId = nodeId; this.logAggPolicy = getLogAggPolicy(conf); this.recoveredLogInitedTime = recoveredLogInitedTime; + logFileSizeThreshold = + conf.getLong(YarnConfiguration.LOG_AGGREGATION_DEBUG_FILESIZE, + YarnConfiguration.DEFAULT_LOG_AGGREGATION_DEBUG_FILESIZE); if (logAggregationFileController == null) { // by default, use T-File Controller this.logAggregationFileController = new LogAggregationTFileController(); @@ -330,6 +333,20 @@ private void uploadLogsForContainers(boolean appFinished) uploadedLogsInThisCycle = true; List uploadedFilePathsInThisCycleList = new ArrayList<>(); uploadedFilePathsInThisCycleList.addAll(uploadedFilePathsInThisCycle); + if (LOG.isDebugEnabled()) { + try { + long fileSize; + for(Path uploadedFilePath : uploadedFilePathsInThisCycleList) { + fileSize = lfs.getFileStatus(uploadedFilePath).getLen(); + if(fileSize >= logFileSizeThreshold) { + LOG.debug("Log File " + uploadedFilePath + + "size is " + fileSize + " bytes"); + } + } + } catch(Exception e1) { + LOG.error("Failed to get log file size " + e1); + } + } deletionTask = new FileDeletionTask(delService, this.userUgi.getShortUserName(), null, uploadedFilePathsInThisCycleList); -- 2.7.4 (Apple Git-66)