From b7a398a5a426ef790377b4edd37d7066f53bf3ad Mon Sep 17 00:00:00 2001 From: Prabhu Joseph Date: Mon, 13 May 2019 19:40:42 +0530 Subject: [PATCH] YARN-9542 --- .../apache/hadoop/yarn/client/cli/TestLogsCLI.java | 50 +++++++++ .../hadoop/yarn/logaggregation/LogCLIHelpers.java | 119 ++++++++++++--------- 2 files changed, 117 insertions(+), 52 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index 801cf40..7d8389b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.client.cli; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -1072,6 +1073,55 @@ public void testFetchApplictionLogsAsAnotherUser() throws Exception { } } + + @Test (timeout = 15000) + public void testGuessAppOwnerWithCustomSuffix() throws Exception { + String remoteLogRootDir = "target/logs/"; + String jobUser = "user1"; + String loggedUser = "user2"; + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); + conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogRootDir); + conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); + + conf.set(YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS, "indexed"); + conf.set("yarn.log-aggregation.file-controller.indexed.class", + "org.apache.hadoop.yarn.logaggregation.filecontroller.ifile" + + ".LogAggregationIndexedFileController"); + conf.set("yarn.log-aggregation.indexed.remote-app-log-dir-suffix", + "indexed"); + + FileSystem fs = FileSystem.get(conf); + + try { + // Test Newer App Log Dir Struture with Custom Suffix + ApplicationId appId1 = ApplicationId.newInstance(0, 1); + Path path = new Path(remoteLogRootDir + jobUser + "/bucket_indexed/0001/" + + appId1); + if (fs.exists(path)) { + fs.delete(path, true); + } + assertTrue(fs.mkdirs(path)); + String appOwner = LogCLIHelpers.getOwnerForAppIdOrNull(appId1, + loggedUser, conf); + assertThat(appOwner).isEqualTo(jobUser); + + // Test Older App Log Dir Struture with Custom Suffix + ApplicationId appId2 = ApplicationId.newInstance(0, 2); + path = new Path(remoteLogRootDir + jobUser + "/indexed/" + appId2); + if (fs.exists(path)) { + fs.delete(path, true); + } + assertTrue(fs.mkdirs(path)); + appOwner = LogCLIHelpers.getOwnerForAppIdOrNull(appId2, loggedUser, conf); + assertThat(appOwner).isEqualTo(jobUser); + + } finally { + fs.delete(new Path(remoteLogRootDir), true); + } + } + @Test (timeout = 5000) public void testLogsCLIWithInvalidArgs() throws Exception { String localDir = "target/SaveLogs"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java index 8a72d80..5630d62 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java @@ -82,71 +82,86 @@ public int dumpAContainersLogs(String appId, String containerId, public static String getOwnerForAppIdOrNull( ApplicationId appId, String bestGuess, Configuration conf) throws IOException { - Path remoteRootLogDir = new Path(conf.get( - YarnConfiguration.NM_REMOTE_APP_LOG_DIR, - YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); - String suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); - Path fullPath = LogAggregationUtils.getRemoteAppLogDir(remoteRootLogDir, - appId, bestGuess, suffix); - - FileContext fc = - FileContext.getFileContext(remoteRootLogDir.toUri(), conf); - String pathAccess = fullPath.toString(); - try { - if (fc.util().exists(fullPath)) { - return bestGuess; - } + LogAggregationFileControllerFactory factory = + new LogAggregationFileControllerFactory(conf); + List fileControllers = factory + .getConfiguredLogAggregationFileControllerList(); + if (fileControllers == null || fileControllers.isEmpty()) { + System.err.println("Can not find any valid fileControllers. " + + " The configurated fileControllers: " + + YarnConfiguration.LOG_AGGREGATION_FILE_FORMATS); + return null; + } + boolean scanOldPath = LogAggregationUtils.isOlderPathEnabled(conf); - boolean scanOldPath = LogAggregationUtils.isOlderPathEnabled(conf); - if (scanOldPath) { - Path olderAppPath = LogAggregationUtils.getOlderRemoteAppLogDir(appId, - bestGuess, remoteRootLogDir, suffix); - if (fc.util().exists(olderAppPath)) { + for (LogAggregationFileController fileFormat : fileControllers) { + Path remoteRootLogDir = fileFormat.getRemoteRootLogDir(); + String suffix = fileFormat.getRemoteRootLogDirSuffix(); + Path fullPath = fileFormat.getRemoteAppLogDir(appId, bestGuess); + + FileContext fc = + FileContext.getFileContext(remoteRootLogDir.toUri(), conf); + String pathAccess = fullPath.toString(); + + try { + if (fc.util().exists(fullPath)) { return bestGuess; } - } - - Path toMatch = LogAggregationUtils. - getRemoteAppLogDir(remoteRootLogDir, appId, "*", suffix); - pathAccess = toMatch.toString(); - FileStatus[] matching = fc.util().globStatus(toMatch); - if (matching == null || matching.length != 1) { if (scanOldPath) { - toMatch = LogAggregationUtils.getOlderRemoteAppLogDir(appId, "*", - remoteRootLogDir, suffix); - try { - matching = fc.util().globStatus(toMatch); - if (matching != null && matching.length == 1) { - //fetch the user from the old path /app-logs/user[/suffix]/app_id - Path parent = matching[0].getPath().getParent(); - //skip the suffix too - if (suffix != null && !StringUtils.isEmpty(suffix)) { - parent = parent.getParent(); - } - return parent.getName(); - } - } catch (IOException e) { - // Ignore IOException from accessing older app log dir + Path olderAppPath = fileFormat.getOlderRemoteAppLogDir(appId, + bestGuess); + pathAccess = olderAppPath.toString(); + if (fc.util().exists(olderAppPath)) { + return bestGuess; } } + } catch (AccessControlException | AccessDeniedException ex) { + logDirNoAccessPermission(pathAccess, bestGuess, ex.getMessage()); return null; } - //fetch the user from the full path /app-logs/user[/suffix]/bucket/app_id - Path parent = matching[0].getPath().getParent(); - //skip the suffix too - if (suffix != null && !StringUtils.isEmpty(suffix)) { - parent = parent.getParent(); + + try { + Path toMatch = fileFormat.getRemoteAppLogDir(appId, null); + FileStatus[] matching = fc.util().globStatus(toMatch); + if (matching != null && matching.length == 1) { + //fetch user from new path /app-logs/user[/suffix]/bucket/app_id + Path parent = matching[0].getPath().getParent(); + //skip the suffix too + if (suffix != null && !StringUtils.isEmpty(suffix)) { + parent = parent.getParent(); + } + //skip the bucket + parent = parent.getParent(); + return parent.getName(); + } + } catch (IOException e) { + // Ignore IOException thrown from wrong file format + } + + if (scanOldPath) { + try { + Path toMatch = fileFormat.getOlderRemoteAppLogDir(appId, null); + FileStatus[] matching = fc.util().globStatus(toMatch); + if (matching != null && matching.length == 1) { + //fetch user from old path /app-logs/user[/suffix]/app_id + Path parent = matching[0].getPath().getParent(); + //skip the suffix too + if (suffix != null && !StringUtils.isEmpty(suffix)) { + parent = parent.getParent(); + } + return parent.getName(); + } + } catch (IOException e) { + // Ignore IOException thrown from wrong file format + } } - //skip the bucket - parent = parent.getParent(); - return parent.getName(); - } catch (AccessControlException | AccessDeniedException ex) { - logDirNoAccessPermission(pathAccess, bestGuess, ex.getMessage()); - return null; } + + return null; } + @Private @VisibleForTesting public int dumpAContainerLogsForLogType(ContainerLogsRequest options) -- 2.7.4 (Apple Git-66)