From 045cfbd72767162cc0edbff0cb4684898a4b5d03 Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Wed, 29 May 2019 13:28:45 +0530 Subject: [PATCH] YARN-9542 --- .../apache/hadoop/yarn/client/cli/TestLogsCLI.java | 11 +++-- .../AggregatedLogDeletionService.java | 12 +++-- .../yarn/logaggregation/LogAggregationUtils.java | 48 ------------------- .../hadoop/yarn/logaggregation/LogCLIHelpers.java | 54 +++++++++++++++++----- .../tfile/LogAggregationTFileController.java | 8 ++-- .../tfile/TFileAggregatedLogsBlock.java | 9 +++- .../TestAggregatedLogDeletionService.java | 43 ++++++++++++----- .../logaggregation/TestAggregatedLogsBlock.java | 2 +- .../logaggregation/TestContainerLogsUtils.java | 9 ++-- .../logaggregation/TestLogAggregationService.java | 8 ++-- 10 files changed, 111 insertions(+), 93 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 5366769..791f6b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -370,7 +370,7 @@ public void testFetchFinishedApplictionLogs() throws Exception { Path path = new Path(remoteLogRootDir + ugi.getShortUserName() - + "/logs/application_0_0001"); + + "/logs-tfile/application_0_0001"); if (fs.exists(path)) { fs.delete(path, true); } @@ -931,7 +931,8 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception { // create the remote app dir for app // but for a different user testUser" - Path path = new Path(remoteLogRootDir + testUser + "/logs/" + appId); + Path path = new Path(remoteLogRootDir + testUser + "/logs-tfile/" + + appId); if (fs.exists(path)) { fs.delete(path, true); } @@ -997,7 +998,7 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception { System.currentTimeMillis(), 1000); String priorityUser = "priority"; Path pathWithoutPerm = new Path(remoteLogRootDir + priorityUser - + "/logs/" + appTest); + + "/logs-tfile/" + appTest); if (fs.exists(pathWithoutPerm)) { fs.delete(pathWithoutPerm, true); } @@ -1355,7 +1356,7 @@ public void testFetchApplictionLogsHar() throws Exception { assertNotNull(harUrl); Path path = new Path(remoteLogRootDir + ugi.getShortUserName() - + "/logs/application_1440536969523_0001"); + + "/logs-tfile/application_1440536969523_0001"); if (fs.exists(path)) { fs.delete(path, true); } @@ -1416,7 +1417,7 @@ private void createContainerLogs(Configuration configuration, } Path path = new Path(remoteLogRootDir + ugi.getShortUserName() - + "/logs/application_0_0001"); + + "/logs-tfile/application_0_0001"); if (fs.exists(path)) { fs.delete(path, true); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java index 841b870..b251862 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java @@ -41,6 +41,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController; +import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; import org.apache.hadoop.yarn.util.ConverterUtils; import com.google.common.annotations.VisibleForTesting; @@ -66,10 +68,12 @@ public LogDeletionTask(Configuration conf, long retentionSecs, ApplicationClientProtocol rmClient) { this.conf = conf; this.retentionMillis = retentionSecs * 1000; - this.suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); - this.remoteRootLogDir = - new Path(conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, - YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); + LogAggregationFileControllerFactory factory = + new LogAggregationFileControllerFactory(conf); + LogAggregationFileController fileController = + factory.getFileControllerForWrite(); + this.remoteRootLogDir = fileController.getRemoteRootLogDir(); + this.suffix = fileController.getRemoteRootLogDirSuffix(); this.rmClient = rmClient; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java index edf2cf3..21994f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogAggregationUtils.java @@ -94,16 +94,6 @@ public static Path getRemoteLogUserDir(Path remoteRootLogDir, String user) { return new Path(remoteRootLogDir, user); } - /** - * Returns the suffix component of the log dir. - * @param conf the configuration - * @return the suffix which will be appended to the user log dir. - */ - public static String getRemoteNodeLogDirSuffix(Configuration conf) { - return conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, - YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR_SUFFIX); - } - /** * Converts a nodeId to a form used in the app log file name. @@ -120,24 +110,6 @@ public static String getNodeString(String nodeId) { return nodeId.toString().replace(":", "_"); } - /** - * Return the remote application log directory. - * @param conf the configuration - * @param appId the application - * @param appOwner the application owner - * @return the remote application log directory path - * @throws IOException if we can not find remote application log directory - */ - public static org.apache.hadoop.fs.Path getRemoteAppLogDir( - Configuration conf, ApplicationId appId, String appOwner) - throws IOException { - String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); - org.apache.hadoop.fs.Path remoteRootLogDir = - new org.apache.hadoop.fs.Path(conf.get( - YarnConfiguration.NM_REMOTE_APP_LOG_DIR, - YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); - return getRemoteAppLogDir(conf, appId, appOwner, remoteRootLogDir, suffix); - } /** * Return the remote application log directory. @@ -221,24 +193,4 @@ public static String getNodeString(String nodeId) { qualifiedLogDir.toUri(), conf).util().listStatus(remoteAppLogDir))); return nodeFiles; } - - /** - * Get all available log files under remote app log directory. - * @param conf the configuration - * @param appId the applicationId - * @param appOwner the application owner - * @return the iterator of available log files - * @throws IOException if there is no log file available - */ - public static RemoteIterator getRemoteNodeFileDir( - Configuration conf, ApplicationId appId, String appOwner) - throws IOException { - Path remoteAppLogDir = getRemoteAppLogDir(conf, appId, appOwner); - RemoteIterator nodeFiles = null; - Path qualifiedLogDir = - FileContext.getFileContext(conf).makeQualified(remoteAppLogDir); - nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(), - conf).listStatus(remoteAppLogDir); - return nodeFiles; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index 9dae7b9..a5db0d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -82,12 +82,41 @@ public int dumpAContainersLogs(String appId, String containerId, public static String getOwnerForAppIdOrNull( ApplicationId appId, String bestGuess, Configuration conf) throws IOException { - Path remoteRootLogDir = new Path(conf.get( - YarnConfiguration.NM_REMOTE_APP_LOG_DIR, - YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); - String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); - Path fullPath = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir, - appId, bestGuess, suffix); + LogAggregationFileControllerFactory factory = + new LogAggregationFileControllerFactory(conf); + List fileControllers = factory + .getConfiguredLogAggregationFileControllerList(); + + if (fileControllers != null && !fileControllers.isEmpty()) { + String owner = null; + for (LogAggregationFileController fileFormat : fileControllers) { + try { + owner = guessOwnerWithFileFormat(fileFormat, appId, bestGuess, conf); + if (owner != null) { + return owner; + } + } + catch (AccessControlException | AccessDeniedException ex) { + return null; + } + catch (IOException io) { + // Ignore IOException thrown from wrong file format + } + } + } else { + System.err.println("Can not find any valid fileControllers. " + + " The configurated fileControllers: " + + YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS); + } + return null; + } + + public static String guessOwnerWithFileFormat( + LogAggregationFileController fileFormat, ApplicationId appId, + String bestGuess, Configuration conf) throws IOException { + Path remoteRootLogDir = fileFormat.getRemoteRootLogDir(); + String suffix = fileFormat.getRemoteRootLogDirSuffix(); + Path fullPath = fileFormat.getRemoteAppLogDir(appId, bestGuess); FileContext fc = FileContext.getFileContext(remoteRootLogDir.toUri(), conf); String pathAccess = fullPath.toString(); @@ -111,7 +140,7 @@ public static String getOwnerForAppIdOrNull( return parent.getName(); } catch (AccessControlException | AccessDeniedException ex) { logDirNoAccessPermission(pathAccess, bestGuess, ex.getMessage()); - return null; + throw ex; } } @@ -183,7 +212,8 @@ public int dumpAllContainersLogs(ContainerLogsRequest options) } if (!foundAnyLogs) { emptyLogDir(LogAggregationUtils.getRemoteAppLogDir( - conf, options.getAppId(), options.getAppOwner()) + conf, options.getAppId(), options.getAppOwner(), + fc.getRemoteRootLogDir(), fc.getRemoteRootLogDirSuffix()) .toString()); return -1; } @@ -254,11 +284,11 @@ public void printNodesList(ContainerLogsRequest options, appOwner, fileFormat.getRemoteRootLogDir(), fileFormat.getRemoteRootLogDirSuffix()); } catch (FileNotFoundException fnf) { - logDirNotExist(LogAggregationUtils.getRemoteAppLogDir( - conf, appId, appOwner).toString()); + logDirNotExist(fileFormat.getRemoteAppLogDir(appId, + appOwner).toString()); } catch (AccessControlException | AccessDeniedException ace) { - logDirNoAccessPermission(LogAggregationUtils.getRemoteAppLogDir( - conf, appId, appOwner).toString(), appOwner, + logDirNoAccessPermission(fileFormat.getRemoteAppLogDir( + appId, appOwner).toString(), appOwner, ace.getMessage()); } if (nodeFiles == null) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java index 561aec4..8d5338a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/LogAggregationTFileController.java @@ -173,7 +173,8 @@ public boolean readAggregatedLogs(ContainerLogsRequest logRequest, || containerIdStr.isEmpty()); long size = logRequest.getBytes(); RemoteIterator nodeFiles = LogAggregationUtils - .getRemoteNodeFileDir(conf, appId, logRequest.getAppOwner()); + .getRemoteNodeFileDir(conf, appId, logRequest.getAppOwner(), + remoteRootLogDir, remoteRootLogDirSuffix); byte[] buf = new byte[65535]; while (nodeFiles != null && nodeFiles.hasNext()) { final FileStatus thisNodeFile = nodeFiles.next(); @@ -267,7 +268,8 @@ public boolean readAggregatedLogs(ContainerLogsRequest logRequest, String nodeIdStr = (nodeId == null) ? null : LogAggregationUtils.getNodeString(nodeId); RemoteIterator nodeFiles = LogAggregationUtils - .getRemoteNodeFileDir(conf, appId, appOwner); + .getRemoteNodeFileDir(conf, appId, appOwner, + remoteRootLogDir, remoteRootLogDirSuffix); if (nodeFiles == null) { throw new IOException("There is no available log file for " + "application:" + appId); @@ -331,7 +333,7 @@ public boolean readAggregatedLogs(ContainerLogsRequest logRequest, @Override public void renderAggregatedLogsBlock(Block html, ViewContext context) { TFileAggregatedLogsBlock block = new TFileAggregatedLogsBlock( - context, conf); + context, conf, remoteRootLogDir, remoteRootLogDirSuffix); block.render(html); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/TFileAggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/TFileAggregatedLogsBlock.java index 6fb5b90..2c2604b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/TFileAggregatedLogsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/tfile/TFileAggregatedLogsBlock.java @@ -48,11 +48,16 @@ public class TFileAggregatedLogsBlock extends LogAggregationHtmlBlock { private final Configuration conf; + private Path remoteRootLogDir; + private String remoteRootLogDirSuffix; @Inject - public TFileAggregatedLogsBlock(ViewContext ctx, Configuration conf) { + public TFileAggregatedLogsBlock(ViewContext ctx, Configuration conf, + Path remoteRootLogDir, String remoteRootLogDirSuffix) { super(ctx); this.conf = conf; + this.remoteRootLogDir = remoteRootLogDir; + this.remoteRootLogDirSuffix = remoteRootLogDirSuffix; } @Override @@ -67,7 +72,7 @@ protected void render(Block html) { try { nodeFiles = LogAggregationUtils .getRemoteNodeFileDir(conf, params.getAppId(), - params.getAppOwner()); + params.getAppOwner(), remoteRootLogDir, remoteRootLogDirSuffix); } catch (RuntimeException e) { throw e; } catch (Exception ex) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java index f36ebf4..c19a4f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.logaggregation; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.LOG_AGGREGATION_FILE_CONTROLLER_FMT; + import java.io.IOException; import java.net.URI; import java.util.Arrays; @@ -37,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.logaggregation.filecontroller.tfile.LogAggregationTFileController; import org.junit.Before; import org.junit.Test; import org.junit.Assert; @@ -59,13 +62,17 @@ public void testDeletion() throws Exception { String root = "mockfs://foo/"; String remoteRootLogDir = root+"tmp/logs"; - String suffix = "logs"; + String configuredSuffix = "logs"; + String actualSuffix = "logs-tfile"; final Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); conf.set(YarnConfiguration.LOG_AGGREGATION_ENABLED, "true"); conf.set(YarnConfiguration.LOG_AGGREGATION_RETAIN_SECONDS, "1800"); conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteRootLogDir); - conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, suffix); + conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, configuredSuffix); + conf.set(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS, "TFile"); + conf.set(String.format(LOG_AGGREGATION_FILE_CONTROLLER_FMT, "TFile"), + LogAggregationTFileController.class.getName()); Path rootPath = new Path(root); FileSystem rootFs = rootPath.getFileSystem(conf); @@ -81,7 +88,7 @@ public void testDeletion() throws Exception { ApplicationId appId1 = ApplicationId.newInstance(System.currentTimeMillis(), 1); - Path userLogDir = new Path(userDir, suffix); + Path userLogDir = new Path(userDir, actualSuffix); Path app1Dir = new Path(userLogDir, appId1.toString()); FileStatus app1DirStatus = new FileStatus(0, true, 0, 0, toDeleteTime, app1Dir); @@ -197,7 +204,8 @@ public void testRefreshLogRetentionSettings() throws Exception { long before50Secs = now - (50 * 1000); String root = "mockfs://foo/"; String remoteRootLogDir = root + "tmp/logs"; - String suffix = "logs"; + String configuredSuffix = "logs"; + String actualSuffix = "logs-tfile"; final Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); conf.set(YarnConfiguration.LOG_AGGREGATION_ENABLED, "true"); @@ -205,7 +213,10 @@ public void testRefreshLogRetentionSettings() throws Exception { conf.set(YarnConfiguration.LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS, "1"); conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteRootLogDir); - conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, suffix); + conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, configuredSuffix); + conf.set(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS, "TFile"); + conf.set(String.format(LOG_AGGREGATION_FILE_CONTROLLER_FMT, "TFile"), + LogAggregationTFileController.class.getName()); Path rootPath = new Path(root); FileSystem rootFs = rootPath.getFileSystem(conf); @@ -220,7 +231,7 @@ public void testRefreshLogRetentionSettings() throws Exception { when(mockFs.listStatus(remoteRootLogPath)).thenReturn( new FileStatus[] { userDirStatus }); - Path userLogDir = new Path(userDir, suffix); + Path userLogDir = new Path(userDir, actualSuffix); ApplicationId appId1 = ApplicationId.newInstance(System.currentTimeMillis(), 1); @@ -309,14 +320,18 @@ public void testCheckInterval() throws Exception { String root = "mockfs://foo/"; String remoteRootLogDir = root+"tmp/logs"; - String suffix = "logs"; + String configuredSuffix = "logs"; + String actualSuffix = "logs-tfile"; Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); conf.set(YarnConfiguration.LOG_AGGREGATION_ENABLED, "true"); conf.set(YarnConfiguration.LOG_AGGREGATION_RETAIN_SECONDS, "864000"); conf.set(YarnConfiguration.LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS, "1"); conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteRootLogDir); - conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, suffix); + conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, configuredSuffix); + conf.set(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS, "TFile"); + conf.set(String.format(LOG_AGGREGATION_FILE_CONTROLLER_FMT, "TFile"), + LogAggregationTFileController.class.getName()); // prevent us from picking up the same mockfs instance from another test FileSystem.closeAll(); @@ -334,7 +349,7 @@ public void testCheckInterval() throws Exception { ApplicationId appId1 = ApplicationId.newInstance(System.currentTimeMillis(), 1); - Path userLogDir = new Path(userDir, suffix); + Path userLogDir = new Path(userDir, actualSuffix); Path app1Dir = new Path(userLogDir, appId1.toString()); FileStatus app1DirStatus = new FileStatus(0, true, 0, 0, now, app1Dir); @@ -391,7 +406,8 @@ public void testRobustLogDeletion() throws Exception { String root = "mockfs://foo/"; String remoteRootLogDir = root+"tmp/logs"; - String suffix = "logs"; + String configuredSuffix = "logs"; + String actualSuffix = "logs-tfile"; Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); @@ -400,7 +416,10 @@ public void testRobustLogDeletion() throws Exception { conf.set(YarnConfiguration.LOG_AGGREGATION_RETAIN_CHECK_INTERVAL_SECONDS, "1"); conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteRootLogDir); - conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, suffix); + conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, configuredSuffix); + conf.set(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS, "TFile"); + conf.set(String.format(LOG_AGGREGATION_FILE_CONTROLLER_FMT, "TFile"), + LogAggregationTFileController.class.getName()); // prevent us from picking up the same mockfs instance from another test FileSystem.closeAll(); @@ -416,7 +435,7 @@ public void testRobustLogDeletion() throws Exception { when(mockFs.listStatus(remoteRootLogPath)).thenReturn( new FileStatus[]{userDirStatus}); - Path userLogDir = new Path(userDir, suffix); + Path userLogDir = new Path(userDir, actualSuffix); ApplicationId appId1 = ApplicationId.newInstance(System.currentTimeMillis(), 1); Path app1Dir = new Path(userLogDir, appId1.toString()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java index bee34e0..1135f9e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogsBlock.java @@ -378,7 +378,7 @@ private void writeLog(String fileName, String text) throws Exception { @Inject TFileAggregatedLogsBlockForTest(ViewContext ctx, Configuration conf) { - super(ctx, conf); + super(ctx, conf, new Path("target/logs"), "logs"); } public void render(Block html) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java index 4767282..8c0b147 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestContainerLogsUtils.java @@ -35,7 +35,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileController; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerContext; import org.apache.hadoop.yarn.logaggregation.filecontroller.LogAggregationFileControllerFactory; @@ -87,8 +86,12 @@ public static void createContainerLogFileInRemoteFS(Configuration conf, createContainerLogInLocalDir(appLogsDir, containerId, fs, fileName, content); // upload container logs to remote log dir - Path path = new Path(conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR), - user + "/logs/" + appId.toString()); + LogAggregationFileControllerFactory factory = + new LogAggregationFileControllerFactory(conf); + LogAggregationFileController fileController = + factory.getFileControllerForWrite(); + Path path = fileController.getRemoteAppLogDir(appId, user); + if (fs.exists(path) && deleteRemoteLogDir) { fs.delete(path, true); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 8b2e3cc..8c08909 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -688,12 +688,14 @@ public void testRemoteRootLogDirIsCreatedWithCorrectGroupOwner() @Test public void testAppLogDirCreation() throws Exception { - final String logSuffix = "logs"; + final String configuredSuffix = "logs"; + final String actualSuffix = "logs-tfile"; this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath()); - this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, logSuffix); + this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, + configuredSuffix); InlineDispatcher dispatcher = new InlineDispatcher(); dispatcher.init(this.conf); @@ -726,7 +728,7 @@ public LogAggregationFileController getLogAggregationFileController( ApplicationId appId = BuilderUtils.newApplicationId(1, 1); Path userDir = fs.makeQualified(new Path( remoteRootLogDir.getAbsolutePath(), this.user)); - Path suffixDir = new Path(userDir, logSuffix); + Path suffixDir = new Path(userDir, actualSuffix); Path appDir = new Path(suffixDir, appId.toString()); LogAggregationContext contextWithAllContainers = Records.newRecord(LogAggregationContext.class); -- 2.7.4 (Apple Git-66)